


'''simple docstring'''







import unittest

from parameterized import parameterized

from transformers import LlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaTokenizer


class  lowerCAmelCase_						:





  """simple docstring"""



  def __init__(					self     :							int				,							SCREAMING_SNAKE_CASE__     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							List[Any]=1_3				,							SCREAMING_SNAKE_CASE__     :							List[Any]=7				,							SCREAMING_SNAKE_CASE__     :							int=True				,							SCREAMING_SNAKE_CASE__     :							List[str]=True				,							SCREAMING_SNAKE_CASE__     :							Optional[int]=False				,							SCREAMING_SNAKE_CASE__     :							Dict=True				,							SCREAMING_SNAKE_CASE__     :							Optional[int]=9_9				,							SCREAMING_SNAKE_CASE__     :							Optional[int]=3_2				,							SCREAMING_SNAKE_CASE__     :							Dict=5				,							SCREAMING_SNAKE_CASE__     :							int=4				,							SCREAMING_SNAKE_CASE__     :							Optional[int]=3_7				,							SCREAMING_SNAKE_CASE__     :							Any="gelu"				,							SCREAMING_SNAKE_CASE__     :							int=0.1				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]=0.1				,							SCREAMING_SNAKE_CASE__     :							Any=5_1_2				,							SCREAMING_SNAKE_CASE__     :							str=1_6				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]=2				,							SCREAMING_SNAKE_CASE__     :							Any=0.0_2				,							SCREAMING_SNAKE_CASE__     :							str=3				,							SCREAMING_SNAKE_CASE__     :							str=4				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]=None				,							):





       '''simple docstring'''

       __a       =   parent
       __a       =   batch_size
       __a       =   seq_length
       __a       =   is_training
       __a       =   use_input_mask
       __a       =   use_token_type_ids
       __a       =   use_labels
       __a       =   vocab_size
       __a       =   hidden_size
       __a       =   num_hidden_layers
       __a       =   num_attention_heads
       __a       =   intermediate_size
       __a       =   hidden_act
       __a       =   hidden_dropout_prob
       __a       =   attention_probs_dropout_prob
       __a       =   max_position_embeddings
       __a       =   type_vocab_size
       __a       =   type_sequence_label_size
       __a       =   initializer_range
       __a       =   num_labels
       __a       =   num_choices
       __a       =   scope

  def __a						(					self     :							Dict						):





       '''simple docstring'''

       __a       =   ids_tensor([self.batch_size, self.seq_length]				,							self.vocab_size						)

       __a       =   None
       if self.use_input_mask:
            __a       =   random_attention_mask([self.batch_size, self.seq_length]						)

       __a       =   None
       if self.use_token_type_ids:
            __a       =   ids_tensor([self.batch_size, self.seq_length]				,							self.type_vocab_size						)

       __a       =   None
       __a       =   None
       __a       =   None
       if self.use_labels:
            __a       =   ids_tensor([self.batch_size]				,							self.type_sequence_label_size						)
            __a       =   ids_tensor([self.batch_size, self.seq_length]				,							self.num_labels						)
            __a       =   ids_tensor([self.batch_size]				,							self.num_choices						)

       __a       =   self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

  def __a						(					self     :							str						):





       '''simple docstring'''

       return LlamaConfig(
           vocab_size=self.vocab_size				,							hidden_size=self.hidden_size				,							num_hidden_layers=self.num_hidden_layers				,							num_attention_heads=self.num_attention_heads				,							intermediate_size=self.intermediate_size				,							hidden_act=self.hidden_act				,							hidden_dropout_prob=self.hidden_dropout_prob				,							attention_probs_dropout_prob=self.attention_probs_dropout_prob				,							max_position_embeddings=self.max_position_embeddings				,							type_vocab_size=self.type_vocab_size				,							is_decoder=__UpperCamelCase				,							initializer_range=self.initializer_range				,							)

  def __a						(					self     :							int				,							SCREAMING_SNAKE_CASE__     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]				,							SCREAMING_SNAKE_CASE__     :							Any						):





       '''simple docstring'''

       __a       =   LlamaModel(config=__UpperCamelCase						)
       model.to(__UpperCamelCase						)
       model.eval()
       __a       =   model(__UpperCamelCase				,							attention_mask=__UpperCamelCase						)
       __a       =   model(__UpperCamelCase						)
       self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, self.seq_length, self.hidden_size)						)

  def __a						(					self     :							int				,							SCREAMING_SNAKE_CASE__     :							Any				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]				,							SCREAMING_SNAKE_CASE__     :							Any				,							SCREAMING_SNAKE_CASE__     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							Any				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]				,							SCREAMING_SNAKE_CASE__     :							int				,							):





       '''simple docstring'''

       __a       =   True
       __a       =   LlamaModel(__UpperCamelCase						)
       model.to(__UpperCamelCase						)
       model.eval()
       __a       =   model(
           __UpperCamelCase				,							attention_mask=__UpperCamelCase				,							encoder_hidden_states=__UpperCamelCase				,							encoder_attention_mask=__UpperCamelCase				,							)
       __a       =   model(
           __UpperCamelCase				,							attention_mask=__UpperCamelCase				,							encoder_hidden_states=__UpperCamelCase				,							)
       __a       =   model(__UpperCamelCase				,							attention_mask=__UpperCamelCase						)
       self.parent.assertEqual(result.last_hidden_state.shape				,							(self.batch_size, self.seq_length, self.hidden_size)						)

  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							Optional[int]				,							SCREAMING_SNAKE_CASE__     :							Dict				,							SCREAMING_SNAKE_CASE__     :							Dict				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]				,							SCREAMING_SNAKE_CASE__     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							int				,							):





       '''simple docstring'''

       __a       =   LlamaForCausalLM(config=__UpperCamelCase						)
       model.to(__UpperCamelCase						)
       model.eval()
       __a       =   model(__UpperCamelCase				,							attention_mask=__UpperCamelCase				,							labels=__UpperCamelCase						)
       self.parent.assertEqual(result.logits.shape				,							(self.batch_size, self.seq_length, self.vocab_size)						)

  def __a						(					self     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							Any				,							SCREAMING_SNAKE_CASE__     :							int				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]				,							SCREAMING_SNAKE_CASE__     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							Optional[int]				,							SCREAMING_SNAKE_CASE__     :							Any				,							SCREAMING_SNAKE_CASE__     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							str				,							):





       '''simple docstring'''

       __a       =   True
       __a       =   True
       __a       =   LlamaForCausalLM(config=__UpperCamelCase						)
       model.to(__UpperCamelCase						)
       model.eval()

       # first forward pass
       __a       =   model(
           __UpperCamelCase				,							attention_mask=__UpperCamelCase				,							encoder_hidden_states=__UpperCamelCase				,							encoder_attention_mask=__UpperCamelCase				,							use_cache=__UpperCamelCase				,							)
       __a       =   outputs.past_key_values

       # create hypothetical multiple next token and extent to next_input_ids
       __a       =   ids_tensor((self.batch_size, 3)				,							config.vocab_size						)
       __a       =   ids_tensor((self.batch_size, 3)				,							vocab_size=2						)

       # append to next input_ids and
       __a       =   torch.cat([input_ids, next_tokens]				,							dim=-1						)
       __a       =   torch.cat([input_mask, next_mask]				,							dim=-1						)

       __a       =   model(
           __UpperCamelCase				,							attention_mask=__UpperCamelCase				,							encoder_hidden_states=__UpperCamelCase				,							encoder_attention_mask=__UpperCamelCase				,							output_hidden_states=__UpperCamelCase				,							)["""hidden_states"""][0]
       __a       =   model(
           __UpperCamelCase				,							attention_mask=__UpperCamelCase				,							encoder_hidden_states=__UpperCamelCase				,							encoder_attention_mask=__UpperCamelCase				,							past_key_values=__UpperCamelCase				,							output_hidden_states=__UpperCamelCase				,							)["""hidden_states"""][0]

       # select random slice
       __a       =   ids_tensor((1,)				,							output_from_past.shape[-1]						).item()
       __a       =   output_from_no_past[:, -3:, random_slice_idx].detach()
       __a       =   output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]						)

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(__UpperCamelCase				,							__UpperCamelCase				,							atol=1E-3						)						)


  def __a						(					self     :							str						):





       '''simple docstring'''

       __a       =   self.prepare_config_and_inputs()
       (
           (
           __a       
       ) ,						(
           __a       
       ) ,						(
           __a       
       ) ,						(
           __a       
       ) ,						(
           __a       
       ) ,						(
           __a       
       ) ,						(
           __a       
       ) ,						       
       )       =   config_and_inputs
       __a       =   {"""input_ids""": input_ids, """attention_mask""": input_mask}
       return config, inputs_dict


@require_torch
class  lowerCAmelCase_						(       __snake_case		,    __snake_case		,    __snake_case		,    unittest.TestCase      ):





  """simple docstring"""



  a_       :int												=(LlamaModel, LlamaForCausalLM, LlamaForSequenceClassification) if is_torch_available() else ()
  a_       :Union[str, Any]												=(LlamaForCausalLM,) if is_torch_available() else ()
  a_       :Any												=(
      {
          """feature-extraction""": LlamaModel,
          """text-classification""": LlamaForSequenceClassification,
          """text-generation""": LlamaForCausalLM,
          """zero-shot""": LlamaForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  a_       :str												=False
  a_       :Dict												=False

  def __a						(					self     :							Tuple						):





       '''simple docstring'''

       __a       =   LlamaModelTester(self						)
       __a       =   ConfigTester(self				,							config_class=__UpperCamelCase				,							hidden_size=3_7						)

  def __a						(					self     :							Dict						):





       '''simple docstring'''

       self.config_tester.run_common_tests()

  def __a						(					self     :							str						):





       '''simple docstring'''

       __a       =   self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*__UpperCamelCase						)

  def __a						(					self     :							Tuple						):





       '''simple docstring'''

       __a       =   self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
            __a       =   type
            self.model_tester.create_and_check_model(*__UpperCamelCase						)

  def __a						(					self     :							Union[str, Any]						):





       '''simple docstring'''

       __a ,						__a       =   self.model_tester.prepare_config_and_inputs_for_common()
       __a       =   3
       __a       =   input_dict["""input_ids"""]
       __a       =   input_ids.ne(1						).to(__UpperCamelCase						)
       __a       =   ids_tensor([self.model_tester.batch_size]				,							self.model_tester.type_sequence_label_size						)
       __a       =   LlamaForSequenceClassification(__UpperCamelCase						)
       model.to(__UpperCamelCase						)
       model.eval()
       __a       =   model(__UpperCamelCase				,							attention_mask=__UpperCamelCase				,							labels=__UpperCamelCase						)
       self.assertEqual(result.logits.shape				,							(self.model_tester.batch_size, self.model_tester.num_labels)						)

  def __a						(					self     :							List[Any]						):





       '''simple docstring'''

       __a ,						__a       =   self.model_tester.prepare_config_and_inputs_for_common()
       __a       =   3
       __a       =   """single_label_classification"""
       __a       =   input_dict["""input_ids"""]
       __a       =   input_ids.ne(1						).to(__UpperCamelCase						)
       __a       =   ids_tensor([self.model_tester.batch_size]				,							self.model_tester.type_sequence_label_size						)
       __a       =   LlamaForSequenceClassification(__UpperCamelCase						)
       model.to(__UpperCamelCase						)
       model.eval()
       __a       =   model(__UpperCamelCase				,							attention_mask=__UpperCamelCase				,							labels=__UpperCamelCase						)
       self.assertEqual(result.logits.shape				,							(self.model_tester.batch_size, self.model_tester.num_labels)						)

  def __a						(					self     :							Union[str, Any]						):





       '''simple docstring'''

       __a ,						__a       =   self.model_tester.prepare_config_and_inputs_for_common()
       __a       =   3
       __a       =   """multi_label_classification"""
       __a       =   input_dict["""input_ids"""]
       __a       =   input_ids.ne(1						).to(__UpperCamelCase						)
       __a       =   ids_tensor(
           [self.model_tester.batch_size, config.num_labels]				,							self.model_tester.type_sequence_label_size						).to(torch.float						)
       __a       =   LlamaForSequenceClassification(__UpperCamelCase						)
       model.to(__UpperCamelCase						)
       model.eval()
       __a       =   model(__UpperCamelCase				,							attention_mask=__UpperCamelCase				,							labels=__UpperCamelCase						)
       self.assertEqual(result.logits.shape				,							(self.model_tester.batch_size, self.model_tester.num_labels)						)

  @unittest.skip("""LLaMA buffers include complex numbers, which breaks this test"""						)
  def __a						(					self     :							Optional[int]						):





       '''simple docstring'''

       pass


  @parameterized.expand([("""linear""",), ("""dynamic""",)]						)
  def __a						(					self     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							Dict						):





       '''simple docstring'''

       __a ,						__a       =   self.model_tester.prepare_config_and_inputs_for_common()
       __a       =   ids_tensor([1, 1_0]				,							config.vocab_size						)
       __a       =   ids_tensor([1, int(config.max_position_embeddings * 1.5						)]				,							config.vocab_size						)

       set_seed(4_2						)  # Fixed seed at init time so the two models get the same random weights
       __a       =   LlamaModel(__UpperCamelCase						)
       original_model.to(__UpperCamelCase						)
       original_model.eval()
       __a       =   original_model(__UpperCamelCase						).last_hidden_state
       __a       =   original_model(__UpperCamelCase						).last_hidden_state

       set_seed(4_2						)  # Fixed seed at init time so the two models get the same random weights
       __a       =   {"""type""": scaling_type, """factor""": 1_0.0}
       __a       =   LlamaModel(__UpperCamelCase						)
       scaled_model.to(__UpperCamelCase						)
       scaled_model.eval()
       __a       =   scaled_model(__UpperCamelCase						).last_hidden_state
       __a       =   scaled_model(__UpperCamelCase						).last_hidden_state

       # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
       # maximum sequence length, so the outputs for the short input should match.
       if scaling_type == "dynamic":
            self.assertTrue(torch.allclose(__UpperCamelCase				,							__UpperCamelCase				,							atol=1E-5						)						)
       else:
            self.assertFalse(torch.allclose(__UpperCamelCase				,							__UpperCamelCase				,							atol=1E-5						)						)

       # The output should be different for long inputs
       self.assertFalse(torch.allclose(__UpperCamelCase				,							__UpperCamelCase				,							atol=1E-5						)						)


@require_torch
class  lowerCAmelCase_						(       unittest.TestCase      ):





  """simple docstring"""



  @unittest.skip("""Logits are not exactly the same, once we fix the instabalities somehow, will update!"""						)
  @slow
  def __a						(					self     :							List[Any]						):





       '''simple docstring'''

       __a       =   [1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
       __a       =   LlamaForCausalLM.from_pretrained("""meta-llama/Llama-2-7b-hf"""				,							device_map="""auto"""						)
       __a       =   model(torch.tensor([input_ids]						)						)
       # Expected mean on dim = -1
       __a       =   torch.tensor([[-6.6_5_5_0, -4.1_2_2_7, -4.9_8_5_9, -3.2_4_0_6, 0.8_2_6_2, -3.0_0_3_3, 1.2_9_6_4, -3.3_6_9_9]]						)
       torch.testing.assert_close(out.mean(-1						)				,							__UpperCamelCase				,							atol=1E-2				,							rtol=1E-2						)
       # slicing logits[0, 0, 0:30]
       # fmt: off
       __a       =   torch.tensor([-1_2.8_2_8_1, -7.4_4_5_3, -0.4_6_3_9, -8.0_6_2_5, -7.2_5_0_0, -8.0_0_0_0, -6.4_8_8_3, -7.7_6_9_5, -7.8_4_3_8, -7.0_3_1_2, -6.2_1_8_8, -7.1_3_2_8, -1.8_4_9_6, 1.9_9_6_1, -8.6_2_5_0, -6.7_2_2_7, -1_2.8_2_8_1, -6.9_4_9_2, -7.0_7_4_2, -7.7_8_5_2, -7.5_8_2_0, -7.9_0_6_2, -6.9_3_7_5, -7.9_8_0_5, -8.3_4_3_8, -8.1_5_6_2, -8.0_4_6_9, -7.6_2_5_0, -7.7_4_2_2, -7.3_3_9_8,]						)
       # fmt: on
       torch.testing.assert_close(out[0, 0, :3_0]				,							__UpperCamelCase				,							atol=1E-5				,							rtol=1E-5						)

  @unittest.skip("""Logits are not exactly the same, once we fix the instabalities somehow, will update!"""						)
  @slow
  def __a						(					self     :							List[Any]						):





       '''simple docstring'''

       __a       =   [1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
       __a       =   LlamaForCausalLM.from_pretrained("""meta-llama/Llama-2-13b-hf"""				,							device_map="""auto"""						)
       __a       =   model(torch.tensor(__UpperCamelCase						)						)
       # Expected mean on dim = -1
       __a       =   torch.tensor([[-2.0_6_2_2, -1.2_7_9_4, -1.1_6_3_8, -0.9_7_8_8, -1.4_6_0_3, -1.0_2_3_8, -1.7_8_9_3, -1.4_4_1_1]]						)
       torch.testing.assert_close(out.mean(-1						)				,							__UpperCamelCase				,							atol=1E-2				,							rtol=1E-2						)
       # slicing logits[0, 0, 0:30]
       # fmt: off
       __a       =   torch.tensor([-8.1_4_0_6, -8.0_5_4_7, 2.7_4_6_1, -1.2_3_4_4, -0.1_4_4_8, -1.8_2_6_2, -1.0_0_2_0, -1.8_1_5_4, -1.6_8_9_5, -1.8_5_1_6, -2.3_5_7_4, -0.9_2_7_7, 3.7_5_9_8, 6.5_7_4_2, -1.2_9_9_8, -0.1_1_7_7, -8.1_4_0_6, -2.9_6_8_8, -2.9_1_9_9, -3.1_6_9_9, -3.5_2_5_4, -2.3_5_5_5, -2.7_9_8_8, -3.4_1_4_1, -2.8_2_6_2, -4.5_1_9_5, -3.3_3_7_9, -3.3_1_6_4, -2.7_8_3_2, -3.0_2_7_3]						)
       # fmt: on
       torch.testing.assert_close(out[0, 0, :3_0]				,							__UpperCamelCase				,							atol=1E-5				,							rtol=1E-5						)

  @unittest.skip("""Logits are not exactly the same, once we fix the instabalities somehow, will update!"""						)
  @slow
  def __a						(					self     :							Optional[Any]						):





       '''simple docstring'''

       __a       =   [1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
       __a       =   LlamaForCausalLM.from_pretrained("""meta-llama/Llama-2-13b-chat-hf"""				,							device_map="""auto"""						)
       __a       =   model(torch.tensor(__UpperCamelCase						)						)
       # Expected mean on dim = -1
       __a       =   torch.tensor([[-0.8_5_6_2, -1.8_5_2_0, -0.7_5_5_1, -0.4_1_6_2, -1.5_1_6_1, -1.2_0_3_8, -2.4_8_2_3, -2.3_2_5_4]]						)
       torch.testing.assert_close(out.mean(-1						)				,							__UpperCamelCase				,							atol=1E-2				,							rtol=1E-2						)
       # slicing logits[0, 0, 0:30]
       # fmt: off
       __a       =   torch.tensor([-2.2_2_2_7, 4.8_8_2_8, 0.9_0_2_3, -0.4_5_7_8, -0.7_8_7_1, -0.1_0_3_3, -0.6_2_2_1, -0.5_7_8_6, -0.7_8_0_3, -1.0_6_7_4, -1.2_9_2_0, -0.1_5_7_0, 0.8_0_0_8, 2.0_7_2_3, -0.9_4_9_7, 0.2_7_7_1, -2.2_2_2_7, -0.7_6_1_2, -1.4_3_4_6, -1.2_0_6_1, -1.6_4_2_6, -0.3_0_0_0, -0.7_1_3_9, -1.1_9_3_4, -1.8_6_9_1, -1.6_9_7_3, -1.5_9_4_7, -1.2_7_0_5, -0.3_5_2_3, -0.5_5_1_3]						)
       # fmt: on
       torch.testing.assert_close(out.mean(-1						)				,							__UpperCamelCase				,							atol=1E-2				,							rtol=1E-2						)

  @unittest.skip(
      """Logits are not exactly the same, once we fix the instabalities somehow, will update! Also it is gonna be a `too_slow` test"""						)
  @slow
  def __a						(					self     :							Optional[int]						):





       '''simple docstring'''

       __a       =   [1, 3_0_6, 4_6_5_8, 2_7_8, 6_5_9_3, 3_1_0, 2_8_3_4, 3_3_8]
       __a       =   LlamaForCausalLM.from_pretrained("""meta-llama/Llama-2-70b-hf"""				,							device_map="""auto"""						)
       __a       =   model(torch.tensor(__UpperCamelCase						)						)

       __a       =   torch.tensor(
           [[-4.2_3_2_7, -3.3_3_6_0, -4.6_6_6_5, -4.7_6_3_1, -1.8_1_8_0, -3.4_1_7_0, -1.4_2_1_1, -3.1_8_1_0]]				,							dtype=torch.floataa						)
       torch.testing.assert_close(out.mean(-1						)				,							__UpperCamelCase				,							atol=1E-2				,							rtol=1E-2						)
       # fmt: off
       __a       =   torch.tensor([-9.4_9_2_2, -3.9_5_5_1, 1.7_9_9_8, -5.6_7_5_8, -5.1_0_5_5, -5.8_9_8_4, -4.8_3_2_0, -6.8_0_8_6, -6.5_3_9_1, -5.6_1_7_2, -5.5_8_2_0, -5.5_3_5_2, 1.7_8_8_1, 3.6_2_8_9, -6.5_1_1_7, -3.4_7_8_5, -9.5_0_0_0, -6.0_3_5_2, -6.8_1_2_5, -6.0_1_9_5, -6.6_8_3_6, -5.4_7_2_7, -6.2_8_1_2, -6.0_3_9_1, -7.3_3_9_8, -7.4_2_9_7, -7.4_8_4_4, -6.5_8_2_0, -5.8_7_8_9, -5.5_3_1_2]						)
       # fmt: on
       torch.testing.assert_close(out[0, 0, :3_0]				,							__UpperCamelCase				,							atol=1E-5				,							rtol=1E-5						)


  @unittest.skip("""Model is curently gated"""						)
  @slow
  def __a						(					self     :							Dict						):





       '''simple docstring'''

       __a       =   """Simply put, the theory of relativity states that 1) the laws of physics are the same everywhere in the universe and 2) the passage of time and the length of objects can vary depending on the observer\'s frame of reference.\n\nThe first part of the theory, that the laws of physics are the same everywhere, is known as the \"princi"""
       __a       =   """Simply put, the theory of relativity states that """
       __a       =   LlamaTokenizer.from_pretrained("""meta-llama/Llama-2-13b-chat-hf"""						)
       __a       =   tokenizer.encode(__UpperCamelCase				,							return_tensors="""pt"""						)
       __a       =   LlamaForCausalLM.from_pretrained(
           """meta-llama/Llama-2-13b-chat-hf"""				,							device_map="""sequential"""				,							use_safetensors=__UpperCamelCase						)

       # greedy generation outputs
       __a       =   model.generate(__UpperCamelCase				,							max_new_tokens=6_4				,							top_p=__UpperCamelCase				,							temperature=1				,							do_sample=__UpperCamelCase						)
       __a       =   tokenizer.decode(generated_ids[0]				,							skip_special_tokens=__UpperCamelCase						)
       self.assertEqual(__UpperCamelCase				,							__UpperCamelCase						)










'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_      =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_      =			{'vocab_file': 'sentencepiece.model'}

SCREAMING_SNAKE_CASE_      =			{
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
}

SCREAMING_SNAKE_CASE_      =			{
    'google/rembert': 2_56,
}


class  lowerCAmelCase_						(       snake_case__      ):





  """simple docstring"""



  a_       :Tuple												=VOCAB_FILES_NAMES
  a_       :Tuple												=PRETRAINED_VOCAB_FILES_MAP
  a_       :Optional[int]												=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

  def __init__(					self     :							Union[str, Any]				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Tuple=False				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]=True				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]=True				,							SCREAMING_SNAKE_CASE__     :							int="[CLS]"				,							SCREAMING_SNAKE_CASE__     :							Optional[int]="[SEP]"				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]="[UNK]"				,							SCREAMING_SNAKE_CASE__     :							Any="[SEP]"				,							SCREAMING_SNAKE_CASE__     :							Dict="[PAD]"				,							SCREAMING_SNAKE_CASE__     :							Dict="[CLS]"				,							SCREAMING_SNAKE_CASE__     :							Dict="[MASK]"				,							**SCREAMING_SNAKE_CASE__     :							List[str]				,							):





       '''simple docstring'''

       super().__init__(
           do_lower_case=SCREAMING_SNAKE_CASE__				,							remove_space=SCREAMING_SNAKE_CASE__				,							keep_accents=SCREAMING_SNAKE_CASE__				,							bos_token=SCREAMING_SNAKE_CASE__				,							eos_token=SCREAMING_SNAKE_CASE__				,							unk_token=SCREAMING_SNAKE_CASE__				,							sep_token=SCREAMING_SNAKE_CASE__				,							pad_token=SCREAMING_SNAKE_CASE__				,							cls_token=SCREAMING_SNAKE_CASE__				,							mask_token=SCREAMING_SNAKE_CASE__				,							**SCREAMING_SNAKE_CASE__				,							)

       __a       =   do_lower_case
       __a       =   remove_space
       __a       =   keep_accents
       __a       =   vocab_file

       __a       =   spm.SentencePieceProcessor()
       self.sp_model.Load(SCREAMING_SNAKE_CASE__						)

  @property
  def __a						(					self     :							List[str]						):





       '''simple docstring'''

       return len(self.sp_model						)

  def __a						(					self     :							str						):





       '''simple docstring'''

       __a       =   {self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__						): i for i in range(self.vocab_size						)}
       vocab.update(self.added_tokens_encoder						)
       return vocab

  def __getstate__(					self     :							List[Any]						):





       '''simple docstring'''

       __a       =   self.__dict__.copy()
       __a       =   None
       return state

  def __setstate__(					self     :							Dict				,							SCREAMING_SNAKE_CASE__     :							Optional[int]						):





       '''simple docstring'''

       __a       =   d
       __a       =   spm.SentencePieceProcessor()
       self.sp_model.Load(self.vocab_file						)

  def __a						(					self     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							List[str]=False						):





       '''simple docstring'''

       __a       =   self.sp_model.EncodeAsPieces(SCREAMING_SNAKE_CASE__						)
       return pieces

  def __a						(					self     :							Optional[Any]				,							SCREAMING_SNAKE_CASE__     :							int						):





       '''simple docstring'''

       return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE__						)

  def __a						(					self     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							List[Any]						):





       '''simple docstring'''

       return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE__						)

  def __a						(					self     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							str						):





       '''simple docstring'''

       __a       =   self.sp_model.decode_pieces(SCREAMING_SNAKE_CASE__						)
       return out_string

  def __a						(					self     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None						):





       '''simple docstring'''

       __a       =   [self.sep_token_id]
       __a       =   [self.cls_token_id]
       if token_ids_a is None:
            return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep

  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None				,							SCREAMING_SNAKE_CASE__     :							bool = False						):





       '''simple docstring'''

       if already_has_special_tokens:
            if token_ids_a is not None:
                 raise ValueError(
                     """You should not supply a second sequence if the provided sequence of """
                     """ids is already formatted with special tokens for the model."""						)
            return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE__						)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE__						)) + [1]
       return [1] + ([0] * len(SCREAMING_SNAKE_CASE__						)) + [1]

  def __a						(					self     :							Union[str, Any]				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None						):





       '''simple docstring'''

       __a       =   [self.sep_token_id]
       __a       =   [self.cls_token_id]

       if token_ids_a is None:
            return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]


  def __a						(					self     :							int				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Optional[str] = None						):





       '''simple docstring'''

       if not os.path.isdir(SCREAMING_SNAKE_CASE__						):
            logger.error("""Vocabulary path ({}) should be a directory""".format(SCREAMING_SNAKE_CASE__						)						)
            return
       __a       =   os.path.join(
           SCREAMING_SNAKE_CASE__				,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(SCREAMING_SNAKE_CASE__						):
            copyfile(self.vocab_file				,							SCREAMING_SNAKE_CASE__						)

       return (out_vocab_file,)






