




"""simple docstring"""

from __future__ import annotations

import unittest

from transformers import XGLMConfig, XGLMTokenizer, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers.models.xglm.modeling_tf_xglm import (
           TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
           TFXGLMForCausalLM,
           TFXGLMModel,
       )





@require_tf
class 							lowercase_			:



 '''simple docstring'''
 UpperCAmelCase       :	List[Any]     =      XGLMConfig
 UpperCAmelCase       :	List[str]     =      {}
 UpperCAmelCase       :	Dict     =      '''gelu'''

 def __init__(					self			:			Optional[Any]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			int=14       ,				_UpperCAmelCase			:			List[Any]=7       ,				_UpperCAmelCase			:			str=True       ,				_UpperCAmelCase			:			Optional[int]=True       ,				_UpperCAmelCase			:			List[Any]=True       ,				_UpperCAmelCase			:			Any=99       ,				_UpperCAmelCase			:			List[str]=32       ,				_UpperCAmelCase			:			List[Any]=2       ,				_UpperCAmelCase			:			int=4       ,				_UpperCAmelCase			:			List[str]=37       ,				_UpperCAmelCase			:			Any="gelu"       ,				_UpperCAmelCase			:			Union[str, Any]=0.1       ,				_UpperCAmelCase			:			Optional[Any]=0.1       ,				_UpperCAmelCase			:			int=512       ,				_UpperCAmelCase			:			List[str]=0.02       ,				):
       _A     =  parent
       _A     =  batch_size
       _A     =  seq_length
       _A     =  is_training
       _A     =  use_input_mask
       _A     =  use_labels
       _A     =  vocab_size
       _A     =  d_model
       _A     =  num_hidden_layers
       _A     =  num_attention_heads
       _A     =  ffn_dim
       _A     =  activation_function
       _A     =  activation_dropout
       _A     =  attention_dropout
       _A     =  max_position_embeddings
       _A     =  initializer_range
       _A     =  None
       _A     =  0
       _A     =  2
       _A     =  1

 def      lowerCAmelCase_				(					self			:			Any						):
       return XGLMConfig.from_pretrained('facebook/xglm-564M'						)

 def      lowerCAmelCase_				(					self			:			str						):
       _A     =  tf.clip_by_value(
           ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size						)       ,				clip_value_min=0       ,				clip_value_max=3						)

       _A     =  None
       if self.use_input_mask:
             _A     =  random_attention_mask([self.batch_size, self.seq_length]						)

       _A     =  self.get_config()

       _A     =  floats_tensor([self.num_hidden_layers, self.num_attention_heads]       ,				2						)

       return (
           config,
           input_ids,
           input_mask,
           head_mask,
       )

 def      lowerCAmelCase_				(					self			:			Dict						):
       return XGLMConfig(
           vocab_size=self.vocab_size       ,				d_model=self.hidden_size       ,				num_layers=self.num_hidden_layers       ,				attention_heads=self.num_attention_heads       ,				ffn_dim=self.ffn_dim       ,				activation_function=self.activation_function       ,				activation_dropout=self.activation_dropout       ,				attention_dropout=self.attention_dropout       ,				max_position_embeddings=self.max_position_embeddings       ,				initializer_range=self.initializer_range       ,				use_cache=_UpperCAmelCase       ,				bos_token_id=self.bos_token_id       ,				eos_token_id=self.eos_token_id       ,				pad_token_id=self.pad_token_id       ,				return_dict=_UpperCAmelCase       ,				)



 def      lowerCAmelCase_				(					self			:			Optional[Any]						):
       _A     =  self.prepare_config_and_inputs()

       (
           (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       
       )     =  config_and_inputs

       _A     =  {
           'input_ids': input_ids,
           'head_mask': head_mask,
       }

       return config, inputs_dict





@require_tf
class 							lowercase_			(	__lowerCAmelCase     ,      __lowerCAmelCase     ,      unittest.TestCase				):



 '''simple docstring'''
 UpperCAmelCase       :	Any     =      (TFXGLMModel, TFXGLMForCausalLM) if is_tf_available() else ()
 UpperCAmelCase       :	List[str]     =      (TFXGLMForCausalLM,) if is_tf_available() else ()
 UpperCAmelCase       :	str     =      (
     {'''feature-extraction''': TFXGLMModel, '''text-generation''': TFXGLMForCausalLM} if is_tf_available() else {}
 )
 UpperCAmelCase       :	Optional[Any]     =      False
 UpperCAmelCase       :	Tuple     =      False
 UpperCAmelCase       :	int     =      False

 def      lowerCAmelCase_				(					self			:			List[Any]						):
       _A     =  TFXGLMModelTester(self						)
       _A     =  ConfigTester(self       ,				config_class=_UpperCAmelCase       ,				n_embd=37						)

 def      lowerCAmelCase_				(					self			:			List[str]						):
       self.config_tester.run_common_tests()

 @slow
 def      lowerCAmelCase_				(					self			:			List[Any]						):
       for model_name in TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             _A     =  TFXGLMModel.from_pretrained(_UpperCAmelCase						)
             self.assertIsNotNone(_UpperCAmelCase						)



 @unittest.skip(reason='Currently, model embeddings are going to undergo a major refactor.'						)
 def      lowerCAmelCase_				(					self			:			List[str]						):
       super().test_resize_token_embeddings()





@require_tf
class 							lowercase_			(	unittest.TestCase				):



 '''simple docstring'''

 @slow
 def      lowerCAmelCase_				(					self			:			Any       ,				_UpperCAmelCase			:			str=True						):
       _A     =  TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'						)
       _A     =  tf.convert_to_tensor([[2, 268, 9_865]]       ,				dtype=tf.intaa						)  # The dog
       # </s> The dog is a very friendly dog. He is very affectionate and loves to play with other
       # fmt: off
       _A     =  [2, 268, 9_865, 67, 11, 1_988, 57_252, 9_865, 5, 984, 67, 1_988, 213_838, 1_658, 53, 70_446, 33, 6_657, 278, 1_581]
       # fmt: on
       _A     =  model.generate(_UpperCAmelCase       ,				do_sample=_UpperCAmelCase       ,				num_beams=1						)
       if verify_outputs:
             self.assertListEqual(output_ids[0].numpy().tolist()       ,				_UpperCAmelCase						)

 @slow
 def      lowerCAmelCase_				(					self			:			List[Any]						):
       _A     =  XGLMTokenizer.from_pretrained('facebook/xglm-564M'						)
       _A     =  TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'						)

       tf.random.set_seed(0						)
       _A     =  tokenizer('Today is a nice day and'       ,				return_tensors='tf'						)
       _A     =  tokenized.input_ids
       # forces the generation to happen on CPU, to avoid GPU-related quirks (and assure same output regardless of the available devices)
       with tf.device(':/CPU:0'						):
             _A     =  model.generate(_UpperCAmelCase       ,				do_sample=_UpperCAmelCase       ,				seed=[7, 0]						)
       _A     =  tokenizer.decode(output_ids[0]       ,				skip_special_tokens=_UpperCAmelCase						)

       _A     =  (
           'Today is a nice day and warm evening here over Southern Alberta!! Today when they closed schools due'
       )
       self.assertEqual(_UpperCAmelCase       ,				_UpperCAmelCase						)



 @slow
 def      lowerCAmelCase_				(					self			:			Union[str, Any]						):
       _A     =  TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'						)
       _A     =  XGLMTokenizer.from_pretrained('facebook/xglm-564M'						)

       _A     =  'left'

       # use different length sentences to test batching
       _A     =  [
           'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
           'left-padding, such as in batched generation. The output for the sequence below should be the same '
           'regardless of whether left padding is applied or not. When',
           'Hello, my dog is a little',
       ]

       _A     =  tokenizer(_UpperCAmelCase       ,				return_tensors='tf'       ,				padding=_UpperCAmelCase						)
       _A     =  inputs['input_ids']

       _A     =  model.generate(input_ids=_UpperCAmelCase       ,				attention_mask=inputs['attention_mask']       ,				max_new_tokens=12						)

       _A     =  tokenizer(sentences[0]       ,				return_tensors='tf'						).input_ids
       _A     =  model.generate(input_ids=_UpperCAmelCase       ,				max_new_tokens=12						)

       _A     =  tokenizer(sentences[1]       ,				return_tensors='tf'						).input_ids
       _A     =  model.generate(input_ids=_UpperCAmelCase       ,				max_new_tokens=12						)

       _A     =  tokenizer.batch_decode(_UpperCAmelCase       ,				skip_special_tokens=_UpperCAmelCase						)
       _A     =  tokenizer.decode(output_non_padded[0]       ,				skip_special_tokens=_UpperCAmelCase						)
       _A     =  tokenizer.decode(output_padded[0]       ,				skip_special_tokens=_UpperCAmelCase						)

       _A     =  [
           'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
           'left-padding, such as in batched generation. The output for the sequence below should be the same '
           'regardless of whether left padding is applied or not. When left padding is applied, the sequence will be '
           'a single',
           'Hello, my dog is a little bit of a shy one, but he is very friendly',
       ]
       self.assertListEqual(_UpperCAmelCase       ,				_UpperCAmelCase						)
       self.assertListEqual(_UpperCAmelCase       ,				[non_padded_sentence, padded_sentence]						)


"""simple docstring"""



from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


_UpperCamelCase								=		logging.get_logger(__name__)

_UpperCamelCase								=		{
    "EleutherAI/gpt-neo-1.3B": "https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json",
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}
class 						__UpperCAmelCase    (__A     ):







			'''simple docstring'''
			_UpperCamelCase  :						Tuple					     =				'gpt_neo'
			_UpperCamelCase  :						Optional[Any]					     =				['past_key_values']
			_UpperCamelCase  :						int					     =				{'num_attention_heads': 'num_heads', 'num_hidden_layers': 'num_layers'}





			def __init__(			self   ,      snake_case_=50_257   ,      snake_case_=2_048   ,      snake_case_=2_048   ,      snake_case_=24   ,      snake_case_=[[["global", "local"], 12]]   ,      snake_case_=16   ,      snake_case_=None   ,      snake_case_=256   ,      snake_case_="gelu_new"   ,      snake_case_=0.0   ,      snake_case_=0.0   ,      snake_case_=0.0   ,      snake_case_=0.1   ,      snake_case_=1E-5   ,      snake_case_=0.02   ,      snake_case_=True   ,      snake_case_=50_256   ,      snake_case_=50_256   ,      **snake_case_   ,      ):







										'''simple docstring'''


										A__       :     int								=			vocab_size
										A__       :     Optional[Any]								=			max_position_embeddings
										A__       :     int								=			hidden_size
										A__       :     List[Any]								=			num_layers
										A__       :     Any								=			num_heads
										A__       :     List[str]								=			intermediate_size
										A__       :     Dict								=			window_size
										A__       :     Optional[int]								=			activation_function
										A__       :     Optional[int]								=			resid_dropout
										A__       :     List[str]								=			embed_dropout
										A__       :     str								=			attention_dropout
										A__       :     List[str]								=			classifier_dropout
										A__       :     str								=			layer_norm_epsilon
										A__       :     str								=			initializer_range
										A__       :     Any								=			use_cache

										A__       :     List[str]								=			bos_token_id
										A__       :     Any								=			eos_token_id

										A__       :     Optional[Any]								=			attention_types
										A__       :     List[Any]								=			self.expand_attention_types_params(snake_case_							)

										if len(self.attention_layers							) != self.num_layers:
																	raise ValueError(
																	    """Configuration for convolutional module is incorrect. """
																	    """It is required that `len(config.attention_layers)` == `config.num_layers` """
																	    F'''but is `len(config.attention_layers) = {len(self.attention_layers							)}`, '''
																	    F'''`config.num_layers = {self.num_layers}`. '''
																	    """`config.attention_layers` is prepared using `config.attention_types`. """
																	    """Please verify the value of `config.attention_types` argument."""							)

										super().__init__(bos_token_id=snake_case_   ,      eos_token_id=snake_case_   ,      **snake_case_							)





			@staticmethod
			def 		lowerCamelCase					(			snake_case_							):







										'''simple docstring'''


										A__       :     Optional[Any]								=			[]
										for item in attention_types:
																	for _ in range(item[1]							):
																								attentions.extend(item[0]							)
										return attentions





def  _A(		lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase							):
							import torch

							A__       :     Union[str, Any]								=			input.size()
							A__       :     str								=			len(lowerCAmelCase							)
							A__       :     Union[str, Any]								=			shape[dimension]

							A__       :     Dict								=			torch.arange(0			,  lowerCAmelCase			,  lowerCAmelCase							)
							A__       :     Dict								=			torch.div(sizedim - size			,  lowerCAmelCase			,  rounding_mode="""floor"""							) + 1
							A__       :     str								=			torch.arange(lowerCAmelCase							) + low_indices[:min_length][:, None]

							A__       :     str								=			[slice(lowerCAmelCase							)] * rank
							A__       :     Optional[Any]								=			indices
							A__       :     Tuple								=			input[s]

							A__       :     List[str]								=			list(range(0			,  rank + 1							)							)
							perm.append(perm.pop(dimension + 1							)							)

							return sliced.permute(lowerCAmelCase							)

def  _A(		lowerCAmelCase			,  lowerCAmelCase							):
							import torch

							A__       :     Optional[Any]								=			torch.arange(1			,  lowerCAmelCase							)
							A__       :     int								=			torch.remainder(lowerCAmelCase			,  lowerCAmelCase							)
							A__       :     str								=			remainders == 0
							A__       :     Tuple								=			candidates[divisor_indices]
							A__       :     Any								=			torch.max(lowerCAmelCase							)
							return largest_divisor, torch.div(lowerCAmelCase			,  lowerCAmelCase			,  rounding_mode="""floor"""							)



class 						__UpperCAmelCase    (__A     ):







			'''simple docstring'''





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Any								=			OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}							)
										if self.use_past:
																	self.fill_with_past_key_values_(snake_case_   ,      direction="""inputs"""							)
																	A__       :     str								=			{0: """batch""", 1: """past_sequence + sequence"""}
										else:
																	A__       :     Optional[Any]								=			{0: """batch""", 1: """sequence"""}

										return common_inputs





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return self._config.num_heads





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = -1   ,      snake_case_ = -1   ,      snake_case_ = False   ,      snake_case_ = None   ,      ):







										'''simple docstring'''


										A__       :     Optional[int]								=			super(snake_case_   ,      self							).generate_dummy_inputs(
										    snake_case_   ,      batch_size=snake_case_   ,      seq_length=snake_case_   ,      is_pair=snake_case_   ,      framework=snake_case_							)

										# We need to order the input in the way they appears in the forward()
										A__       :     List[Any]								=			OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}							)

										# Need to add the past_keys
										if self.use_past:
																	if not is_torch_available():
																								raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""							)
																	else:
																								import torch

																								A__				,							A__       :     Dict								=			common_inputs["""input_ids"""].shape
																								# Not using the same length for past_key_values
																								A__       :     Any								=			seqlen + 2
																								A__       :     Any								=			(
																								    batch,
																								    self.num_attention_heads,
																								    past_key_values_length,
																								    self._config.hidden_size // self.num_attention_heads,
																								)
																								A__       :     List[Any]								=			[
																								    (torch.zeros(snake_case_							), torch.zeros(snake_case_							)) for _ in range(self.num_layers							)
																								]

										A__       :     Tuple								=			common_inputs["""attention_mask"""]
										if self.use_past:
																	A__       :     List[Any]								=			ordered_inputs["""attention_mask"""].dtype
																	A__       :     Optional[Any]								=			torch.cat(
																	    [ordered_inputs["""attention_mask"""], torch.ones(snake_case_   ,      snake_case_   ,      dtype=snake_case_							)]   ,      dim=1							)

										return ordered_inputs





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return 13





