





import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           GPTNeoXForCausalLM,
           GPTNeoXForQuestionAnswering,
           GPTNeoXForSequenceClassification,
           GPTNeoXForTokenClassification,
           GPTNeoXModel,
       )







class       __A							:




      """simple docstring"""







      def __init__(		self   ,				a__   ,				a__=13   ,				a__=7   ,				a__=True   ,				a__=True   ,				a__=True   ,				a__=True   ,				a__=99   ,				a__=64   ,				a__=5   ,				a__=4   ,				a__=37   ,				a__="gelu"   ,				a__=0.1   ,				a__=0.1   ,				a__=512   ,				a__=16   ,				a__=2   ,				a__=0.02   ,				a__=3   ,				a__=4   ,				a__=None   ,				):



             """simple docstring"""





             _lowerCamelCase				:  List[Any]						=		parent
             _lowerCamelCase				:  Tuple						=		batch_size
             _lowerCamelCase				:  str						=		seq_length
             _lowerCamelCase				:  Dict						=		is_training
             _lowerCamelCase				:  Dict						=		use_input_mask
             _lowerCamelCase				:  Tuple						=		use_token_type_ids
             _lowerCamelCase				:  Optional[Any]						=		use_labels
             _lowerCamelCase				:  Optional[Any]						=		vocab_size
             _lowerCamelCase				:  Any						=		hidden_size
             _lowerCamelCase				:  List[str]						=		num_hidden_layers
             _lowerCamelCase				:  Any						=		num_attention_heads
             _lowerCamelCase				:  Dict						=		intermediate_size
             _lowerCamelCase				:  List[Any]						=		hidden_act
             _lowerCamelCase				:  int						=		hidden_dropout_prob
             _lowerCamelCase				:  str						=		attention_probs_dropout_prob
             _lowerCamelCase				:  str						=		max_position_embeddings
             _lowerCamelCase				:  Union[str, Any]						=		type_vocab_size
             _lowerCamelCase				:  Dict						=		type_sequence_label_size
             _lowerCamelCase				:  Dict						=		initializer_range
             _lowerCamelCase				:  str						=		num_labels
             _lowerCamelCase				:  List[str]						=		num_choices
             _lowerCamelCase				:  Union[str, Any]						=		scope
             _lowerCamelCase				:  Tuple						=		vocab_size - 1


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Dict						=		ids_tensor([self.batch_size, self.seq_length]   ,				self.vocab_size)

             _lowerCamelCase				:  Union[str, Any]						=		None
             if self.use_input_mask:
                    _lowerCamelCase				:  Dict						=		random_attention_mask([self.batch_size, self.seq_length])

             _lowerCamelCase				:  Optional[Any]						=		None
             if self.use_labels:
                    _lowerCamelCase				:  Any						=		ids_tensor([self.batch_size, self.seq_length]   ,				self.num_labels)

             _lowerCamelCase				:  int						=		self.get_config()

             return config, input_ids, input_mask, token_labels


      def 					__snake_case      (		self):



             """simple docstring"""





             return GPTNeoXConfig(
                 vocab_size=self.vocab_size   ,				hidden_size=self.hidden_size   ,				num_hidden_layers=self.num_hidden_layers   ,				num_attention_heads=self.num_attention_heads   ,				intermediate_size=self.intermediate_size   ,				hidden_act=self.hidden_act   ,				hidden_dropout_prob=self.hidden_dropout_prob   ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,				max_position_embeddings=self.max_position_embeddings   ,				type_vocab_size=self.type_vocab_size   ,				is_decoder=a__   ,				initializer_range=self.initializer_range   ,				pad_token_id=self.pad_token_id   ,				)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase				:  Optional[int]						=		self.prepare_config_and_inputs()

             _lowerCamelCase				:  Union[str, Any]						=		True

             return config, input_ids, input_mask, token_labels


      def 					__snake_case      (		self   ,				a__   ,				a__   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  List[str]						=		GPTNeoXModel(config=a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase				:  int						=		model(a__   ,				attention_mask=a__)
             _lowerCamelCase				:  str						=		model(a__)
             self.parent.assertEqual(result.last_hidden_state.shape   ,				(self.batch_size, self.seq_length, self.hidden_size))


      def 					__snake_case      (		self   ,				a__   ,				a__   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  List[Any]						=		True
             _lowerCamelCase				:  List[str]						=		GPTNeoXModel(a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase				:  List[Any]						=		model(a__   ,				attention_mask=a__)
             self.parent.assertEqual(result.last_hidden_state.shape   ,				(self.batch_size, self.seq_length, self.hidden_size))


      def 					__snake_case      (		self   ,				a__   ,				a__   ,				a__   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  Dict						=		GPTNeoXForCausalLM(config=a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase				:  Dict						=		model(a__   ,				attention_mask=a__   ,				labels=a__)
             self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.seq_length, self.vocab_size))


      def 					__snake_case      (		self   ,				a__   ,				a__   ,				a__   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  Tuple						=		self.num_labels
             _lowerCamelCase				:  List[str]						=		GPTNeoXForQuestionAnswering(a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase				:  Optional[int]						=		model(a__   ,				attention_mask=a__)
             self.parent.assertEqual(result.start_logits.shape   ,				(self.batch_size, self.seq_length))
             self.parent.assertEqual(result.end_logits.shape   ,				(self.batch_size, self.seq_length))


      def 					__snake_case      (		self   ,				a__   ,				a__   ,				a__   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  List[str]						=		self.num_labels
             _lowerCamelCase				:  List[str]						=		GPTNeoXForSequenceClassification(a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase				:  Union[str, Any]						=		ids_tensor([self.batch_size]   ,				self.type_sequence_label_size)
             _lowerCamelCase				:  List[str]						=		model(a__   ,				attention_mask=a__   ,				labels=a__)
             self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.num_labels))


      def 					__snake_case      (		self   ,				a__   ,				a__   ,				a__   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		self.num_labels
             _lowerCamelCase				:  List[str]						=		GPTNeoXForTokenClassification(a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase				:  Optional[int]						=		model(a__   ,				attention_mask=a__   ,				labels=a__)
             self.parent.assertEqual(result.logits.shape   ,				(self.batch_size, self.seq_length, self.num_labels))


      def 					__snake_case      (		self   ,				a__   ,				a__   ,				a__):



             """simple docstring"""





             _lowerCamelCase				:  List[str]						=		True
             _lowerCamelCase				:  Optional[Any]						=		GPTNeoXForCausalLM(config=a__)
             model.to(a__)
             model.eval()

             # first forward pass
             _lowerCamelCase				:  Dict						=		model(a__   ,				attention_mask=a__   ,				use_cache=a__)
             _lowerCamelCase				:  Any						=		outputs.past_key_values

             # create hypothetical multiple next token and extent to next_input_ids
             _lowerCamelCase				:  List[str]						=		ids_tensor((self.batch_size, 3)   ,				config.vocab_size)
             _lowerCamelCase				:  Dict						=		ids_tensor((self.batch_size, 3)   ,				vocab_size=2)

             # append to next input_ids and
             _lowerCamelCase				:  Tuple						=		torch.cat([input_ids, next_tokens]   ,				dim=-1)
             _lowerCamelCase				:  Tuple						=		torch.cat([input_mask, next_mask]   ,				dim=-1)

             _lowerCamelCase				:  str						=		model(a__   ,				attention_mask=a__   ,				output_hidden_states=a__)
             _lowerCamelCase				:  Union[str, Any]						=		output_from_no_past['''hidden_states'''][0]
             _lowerCamelCase				:  Optional[int]						=		model(
                 a__   ,				attention_mask=a__   ,				past_key_values=a__   ,				output_hidden_states=a__   ,				)['''hidden_states'''][0]

             # select random slice
             _lowerCamelCase				:  Optional[Any]						=		ids_tensor((1,)   ,				output_from_past.shape[-1]).item()
             _lowerCamelCase				:  Tuple						=		output_from_no_past[:, -3:, random_slice_idx].detach()
             _lowerCamelCase				:  List[Any]						=		output_from_past[:, :, random_slice_idx].detach()

             self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

             # test that outputs are equal for slice
             self.parent.assertTrue(torch.allclose(a__   ,				a__   ,				atol=1e-3))



      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		self.prepare_config_and_inputs()
             _lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase				:  Tuple						=		config_and_inputs
             _lowerCamelCase				:  Tuple						=		{'''input_ids''': input_ids, '''attention_mask''': input_mask}
             return config, inputs_dict







@require_torch
class       __A							(   lowerCamelCase__			,lowerCamelCase__			,lowerCamelCase__			,unittest.TestCase	):




      """simple docstring"""







      UpperCAmelCase__							=   (
          (
              GPTNeoXModel,
              GPTNeoXForCausalLM,
              GPTNeoXForQuestionAnswering,
              GPTNeoXForSequenceClassification,
              GPTNeoXForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      UpperCAmelCase__							=   (GPTNeoXForCausalLM,) if is_torch_available() else ()
      UpperCAmelCase__							=   (
          {
              """feature-extraction""": GPTNeoXModel,
              """question-answering""": GPTNeoXForQuestionAnswering,
              """text-classification""": GPTNeoXForSequenceClassification,
              """text-generation""": GPTNeoXForCausalLM,
              """token-classification""": GPTNeoXForTokenClassification,
              """zero-shot""": GPTNeoXForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      UpperCAmelCase__							=   False
      UpperCAmelCase__							=   False
      UpperCAmelCase__							=   False
      UpperCAmelCase__							=   False


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Tuple						=		GPTNeoXModelTester(self)
             _lowerCamelCase				:  List[Any]						=		ConfigTester(self   ,				config_class=a__   ,				hidden_size=64   ,				num_attention_heads=8)


      def 					__snake_case      (		self):



             """simple docstring"""





             self.config_tester.run_common_tests()


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase				:  List[Any]						=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(a__   ,				a__   ,				a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase				:  Optional[int]						=		self.model_tester.prepare_config_and_inputs_for_decoder()
             self.model_tester.create_and_check_model_as_decoder(a__   ,				a__   ,				a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase				:  List[str]						=		self.model_tester.prepare_config_and_inputs_for_decoder()

             _lowerCamelCase				:  Dict						=		None

             self.model_tester.create_and_check_model_as_decoder(a__   ,				a__   ,				a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase				:  Dict						=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_decoder_model_past_large_inputs(a__   ,				a__   ,				a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Union[str, Any]						=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_causal_lm(*a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_question_answering(*a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[int]						=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_sequence_classification(*a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Any						=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_token_classification(*a__)


      @unittest.skip(reason='''Feed forward chunking is not implemented''')
      def 					__snake_case      (		self):



             """simple docstring"""





             pass



      @parameterized.expand([('''linear''',), ('''dynamic''',)])
      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             _lowerCamelCase,	_lowerCamelCase				:  Tuple						=		self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase				:  str						=		ids_tensor([1, 10]   ,				config.vocab_size)
             _lowerCamelCase				:  Any						=		ids_tensor([1, int(config.max_position_embeddings * 1.5)]   ,				config.vocab_size)

             set_seed(42)  # Fixed seed at init time so the two models get the same random weights
             _lowerCamelCase				:  int						=		GPTNeoXModel(a__)
             original_model.to(a__)
             original_model.eval()
             _lowerCamelCase				:  Any						=		original_model(a__).last_hidden_state
             _lowerCamelCase				:  List[Any]						=		original_model(a__).last_hidden_state

             set_seed(42)  # Fixed seed at init time so the two models get the same random weights
             _lowerCamelCase				:  Tuple						=		{'''type''': scaling_type, '''factor''': 10.0}
             _lowerCamelCase				:  Any						=		GPTNeoXModel(a__)
             scaled_model.to(a__)
             scaled_model.eval()
             _lowerCamelCase				:  List[str]						=		scaled_model(a__).last_hidden_state
             _lowerCamelCase				:  Optional[int]						=		scaled_model(a__).last_hidden_state

             # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
             # maximum sequence length, so the outputs for the short input should match.
             if scaling_type == "dynamic":
                    self.assertTrue(torch.allclose(a__   ,				a__   ,				atol=1e-5))
             else:
                    self.assertFalse(torch.allclose(a__   ,				a__   ,				atol=1e-5))

             # The output should be different for long inputs
             self.assertFalse(torch.allclose(a__   ,				a__   ,				atol=1e-5))







@require_torch
class       __A							(   unittest.TestCase	):




      """simple docstring"""







      @slow
      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[int]						=		AutoTokenizer.from_pretrained('''EleutherAI/pythia-410m-deduped''')
             for checkpointing in [True, False]:
                    _lowerCamelCase				:  str						=		GPTNeoXForCausalLM.from_pretrained('''EleutherAI/pythia-410m-deduped''')

                    if checkpointing:
                           model.gradient_checkpointing_enable()
                    else:
                           model.gradient_checkpointing_disable()
                    model.to(a__)

                    _lowerCamelCase				:  str						=		tokenizer('''My favorite food is'''   ,				return_tensors='''pt''').to(a__)
                    # The hub repo. is updated on 2023-04-04, resulting in poor outputs.
                    # See: https://github.com/huggingface/transformers/pull/24193
                    _lowerCamelCase				:  int						=		'''My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI\'m not sure'''

                    _lowerCamelCase				:  Optional[int]						=		model.generate(**a__   ,				do_sample=a__   ,				max_new_tokens=20)
                    _lowerCamelCase				:  Dict						=		tokenizer.batch_decode(a__)[0]

                    self.assertEqual(a__   ,				a__)








import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       __A							(   lowerCamelCase__			,unittest.TestCase	):




      """simple docstring"""







      UpperCAmelCase__							=   OpenAIGPTTokenizer
      UpperCAmelCase__							=   OpenAIGPTTokenizerFast
      UpperCAmelCase__							=   True
      UpperCAmelCase__							=   False


      def 					__snake_case      (		self):



             """simple docstring"""





             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             _lowerCamelCase				:  Tuple						=		[
                 '''l''',
                 '''o''',
                 '''w''',
                 '''e''',
                 '''r''',
                 '''s''',
                 '''t''',
                 '''i''',
                 '''d''',
                 '''n''',
                 '''w</w>''',
                 '''r</w>''',
                 '''t</w>''',
                 '''lo''',
                 '''low''',
                 '''er</w>''',
                 '''low</w>''',
                 '''lowest</w>''',
                 '''newer</w>''',
                 '''wider</w>''',
                 '''<unk>''',
             ]
             _lowerCamelCase				:  Any						=		dict(zip(a__   ,				range(len(a__))))
             _lowerCamelCase				:  List[str]						=		['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

             _lowerCamelCase				:  str						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''vocab_file'''])
             _lowerCamelCase				:  int						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''merges_file'''])
             with open(self.vocab_file   ,				'''w''') as fp:
                    fp.write(json.dumps(a__))
             with open(self.merges_file   ,				'''w''') as fp:
                    fp.write('''\n'''.join(a__))


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             return "lower newer", "lower newer"


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		OpenAIGPTTokenizer(self.vocab_file   ,				self.merges_file)

             _lowerCamelCase				:  Dict						=		'''lower'''
             _lowerCamelCase				:  Optional[Any]						=		['''low''', '''er</w>''']
             _lowerCamelCase				:  List[Any]						=		tokenizer.tokenize(a__)
             self.assertListEqual(a__   ,				a__)

             _lowerCamelCase				:  Any						=		tokens + ['''<unk>''']
             _lowerCamelCase				:  Optional[int]						=		[14, 15, 20]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)   ,				a__)


      def 					__snake_case      (		self   ,				a__=15):



             """simple docstring"""





             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                           _lowerCamelCase				:  Dict						=		self.rust_tokenizer_class.from_pretrained(a__   ,				**a__)

                           # Simple input
                           _lowerCamelCase				:  Optional[int]						=		'''This is a simple input'''
                           _lowerCamelCase				:  Optional[Any]						=		['''This is a simple input 1''', '''This is a simple input 2''']
                           _lowerCamelCase				:  Optional[int]						=		('''This is a simple input''', '''This is a pair''')
                           _lowerCamelCase				:  List[Any]						=		[
                               ('''This is a simple input 1''', '''This is a simple input 2'''),
                               ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                           ]

                           # Simple input tests
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)



      def 					__snake_case      (		self):



             """simple docstring"""





             pass







@require_ftfy
@require_spacy
@require_tokenizers
class       __A							(   lowerCamelCase__	):




      """simple docstring"""







      pass

