






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import numpy as np
					import tensorflow as tf

					from transformers import (
					    TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
					    FlaubertConfig,
					    TFFlaubertForMultipleChoice,
					    TFFlaubertForQuestionAnsweringSimple,
					    TFFlaubertForSequenceClassification,
					    TFFlaubertForTokenClassification,
					    TFFlaubertModel,
					    TFFlaubertWithLMHeadModel,
					)
class      UpperCamelCase :







		"""simple docstring"""



		def __init__(  self		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       Any						,)    ->   List[Any]:


						'''simple docstring'''
						A	 =      parent
						A	 =      1_3
						A	 =      7
						A	 =      True
						A	 =      True
						A	 =      True
						A	 =      True
						A	 =      True
						A	 =      False
						A	 =      False
						A	 =      False
						A	 =      2
						A	 =      9_9
						A	 =      0
						A	 =      3_2
						A	 =      2
						A	 =      4
						A	 =      0.1
						A	 =      0.1
						A	 =      5_1_2
						A	 =      1_6
						A	 =      2
						A	 =      0.02
						A	 =      3
						A	 =      4
						A	 =      'last'
						A	 =      True
						A	 =      None
						A	 =      0
		def       A(  self		:       int		)    ->   Tuple:


						'''simple docstring'''
						A	 =      ids_tensor([self.batch_size, self.seq_length]						,self.vocab_size		)
						A	 =      random_attention_mask([self.batch_size, self.seq_length]						,dtype=tf.floataa		)

						A	 =      None
						if self.use_input_lengths:
										A	 =      (
										    ids_tensor([self.batch_size]						,vocab_size=2		) + self.seq_length - 2
										)  # small variation of seq_length

						A	 =      None
						if self.use_token_type_ids:
										A	 =      ids_tensor([self.batch_size, self.seq_length]						,self.n_langs		)

						A	 =      None
						A	 =      None
						A	 =      None
						if self.use_labels:
										A	 =      ids_tensor([self.batch_size]						,self.type_sequence_label_size		)
										A	 =      ids_tensor([self.batch_size, self.seq_length]						,self.num_labels		)
										A	 =      ids_tensor([self.batch_size]						,2						,dtype=tf.floataa		)
										A	 =      ids_tensor([self.batch_size]						,self.num_choices		)

						A	 =      FlaubertConfig(
						    vocab_size=self.vocab_size						,n_special=self.n_special						,emb_dim=self.hidden_size						,n_layers=self.num_hidden_layers						,n_heads=self.num_attention_heads						,dropout=self.hidden_dropout_prob						,attention_dropout=self.attention_probs_dropout_prob						,gelu_activation=self.gelu_activation						,sinusoidal_embeddings=self.sinusoidal_embeddings						,asm=self.asm						,causal=self.causal						,n_langs=self.n_langs						,max_position_embeddings=self.max_position_embeddings						,initializer_range=self.initializer_range						,summary_type=self.summary_type						,use_proj=self.use_proj						,bos_token_id=self.bos_token_id						,)

						return (
						    config,
						    input_ids,
						    token_type_ids,
						    input_lengths,
						    sequence_labels,
						    token_labels,
						    is_impossible_labels,
						    choice_labels,
						    input_mask,
						)
		def       A(  self		:       Tuple						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,)    ->   Optional[Any]:


						'''simple docstring'''
						A	 =      TFFlaubertModel(config=_SCREAMING_SNAKE_CASE		)
						A	 =      {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
						A	 =      model(_SCREAMING_SNAKE_CASE		)

						A	 =      [input_ids, input_mask]
						A	 =      model(_SCREAMING_SNAKE_CASE		)
						self.parent.assertEqual(result.last_hidden_state.shape						,(self.batch_size, self.seq_length, self.hidden_size)		)
		def       A(  self		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       Any						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       int						,)    ->   Tuple:


						'''simple docstring'''
						A	 =      TFFlaubertWithLMHeadModel(_SCREAMING_SNAKE_CASE		)

						A	 =      {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
						A	 =      model(_SCREAMING_SNAKE_CASE		)

						self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.seq_length, self.vocab_size)		)
		def       A(  self		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       List[str]						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       Dict						,_SCREAMING_SNAKE_CASE		:       str						,)    ->   Tuple:


						'''simple docstring'''
						A	 =      TFFlaubertForQuestionAnsweringSimple(_SCREAMING_SNAKE_CASE		)

						A	 =      {'input_ids': input_ids, 'lengths': input_lengths}

						A	 =      model(_SCREAMING_SNAKE_CASE		)

						self.parent.assertEqual(result.start_logits.shape						,(self.batch_size, self.seq_length)		)
						self.parent.assertEqual(result.end_logits.shape						,(self.batch_size, self.seq_length)		)
		def       A(  self		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,)    ->   Union[str, Any]:


						'''simple docstring'''
						A	 =      TFFlaubertForSequenceClassification(_SCREAMING_SNAKE_CASE		)

						A	 =      {'input_ids': input_ids, 'lengths': input_lengths}

						A	 =      model(_SCREAMING_SNAKE_CASE		)

						self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.type_sequence_label_size)		)
		def       A(  self		:       Any						,_SCREAMING_SNAKE_CASE		:       List[str]						,_SCREAMING_SNAKE_CASE		:       List[str]						,_SCREAMING_SNAKE_CASE		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       List[str]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       List[str]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       str						,)    ->   Tuple:


						'''simple docstring'''
						A	 =      self.num_labels
						A	 =      TFFlaubertForTokenClassification(config=_SCREAMING_SNAKE_CASE		)
						A	 =      {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
						A	 =      model(_SCREAMING_SNAKE_CASE		)
						self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.seq_length, self.num_labels)		)
		def       A(  self		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       List[str]						,_SCREAMING_SNAKE_CASE		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       Any						,_SCREAMING_SNAKE_CASE		:       List[Any]						,_SCREAMING_SNAKE_CASE		:       str						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       Dict						,)    ->   Any:


						'''simple docstring'''
						A	 =      self.num_choices
						A	 =      TFFlaubertForMultipleChoice(config=_SCREAMING_SNAKE_CASE		)
						A	 =      tf.tile(tf.expand_dims(_SCREAMING_SNAKE_CASE						,1		)						,(1, self.num_choices, 1)		)
						A	 =      tf.tile(tf.expand_dims(_SCREAMING_SNAKE_CASE						,1		)						,(1, self.num_choices, 1)		)
						A	 =      tf.tile(tf.expand_dims(_SCREAMING_SNAKE_CASE						,1		)						,(1, self.num_choices, 1)		)
						A	 =      {
						    'input_ids': multiple_choice_inputs_ids,
						    'attention_mask': multiple_choice_input_mask,
						    'token_type_ids': multiple_choice_token_type_ids,
						}
						A	 =      model(_SCREAMING_SNAKE_CASE		)
						self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.num_choices)		)







		def       A(  self		:       int		)    ->   int:


						'''simple docstring'''
						A	 =      self.prepare_config_and_inputs()
						(
						    (
						    A
						) ,		(
						    A
						) ,		(
						    A
						) ,		(
						    A
						) ,		(
						    A
						) ,		(
						    A
						) ,		(
						    A
						) ,		(
						    A
						) ,		(
						    A
						) ,		
						)	 =      config_and_inputs
						A	 =      {
						    'input_ids': input_ids,
						    'token_type_ids': token_type_ids,
						    'langs': token_type_ids,
						    'lengths': input_lengths,
						}
						return config, inputs_dict


@require_tf
class      UpperCamelCase (				snake_case__			,       snake_case__			,       unittest.TestCase ):







		"""simple docstring"""



		snake_case    =   (
		    (
		        TFFlaubertModel,
		        TFFlaubertWithLMHeadModel,
		        TFFlaubertForSequenceClassification,
		        TFFlaubertForQuestionAnsweringSimple,
		        TFFlaubertForTokenClassification,
		        TFFlaubertForMultipleChoice,
		    )
		    if is_tf_available()
		    else ()
		)
		snake_case    =   (
		    (TFFlaubertWithLMHeadModel,) if is_tf_available() else ()
		)  # TODO (PVP): Check other models whether language generation is also applicable
		snake_case    =   (
		    {
		        "feature-extraction": TFFlaubertModel,
		        "fill-mask": TFFlaubertWithLMHeadModel,
		        "question-answering": TFFlaubertForQuestionAnsweringSimple,
		        "text-classification": TFFlaubertForSequenceClassification,
		        "token-classification": TFFlaubertForTokenClassification,
		        "zero-shot": TFFlaubertForSequenceClassification,
		    }
		    if is_tf_available()
		    else {}
		)
		snake_case    =   False
		snake_case    =   False
		def       A(  self		:       str						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       Tuple						,_SCREAMING_SNAKE_CASE		:       List[Any]		)    ->   Optional[int]:


						'''simple docstring'''
						if (
						    pipeline_test_casse_name == "QAPipelineTests"
						    and tokenizer_name is not None
						    and not tokenizer_name.endswith('Fast'		)
						):
										# `QAPipelineTests` fails for a few models when the slower tokenizer are used.
										# (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
										# TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
										return True

						return False
		def       A(  self		:       List[Any]		)    ->   Dict:


						'''simple docstring'''
						A	 =      TFFlaubertModelTester(self		)
						A	 =      ConfigTester(self						,config_class=_SCREAMING_SNAKE_CASE						,emb_dim=3_7		)
		def       A(  self		:       Union[str, Any]		)    ->   Union[str, Any]:


						'''simple docstring'''
						self.config_tester.run_common_tests()
		def       A(  self		:       Optional[int]		)    ->   Optional[int]:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_model(*_SCREAMING_SNAKE_CASE		)
		def       A(  self		:       List[Any]		)    ->   List[str]:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_lm_head(*_SCREAMING_SNAKE_CASE		)
		def       A(  self		:       Tuple		)    ->   Optional[Any]:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_qa(*_SCREAMING_SNAKE_CASE		)
		def       A(  self		:       Any		)    ->   Any:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_sequence_classif(*_SCREAMING_SNAKE_CASE		)
		def       A(  self		:       List[Any]		)    ->   int:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_for_token_classification(*_SCREAMING_SNAKE_CASE		)
		def       A(  self		:       List[str]		)    ->   str:


						'''simple docstring'''
						A	 =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_flaubert_for_multiple_choice(*_SCREAMING_SNAKE_CASE		)







		@slow
		def       A(  self		:       List[Any]		)    ->   Optional[int]:


						'''simple docstring'''
						for model_name in TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										A	 =      TFFlaubertModel.from_pretrained(_SCREAMING_SNAKE_CASE		)
										self.assertIsNotNone(_SCREAMING_SNAKE_CASE		)


@require_tf
@require_sentencepiece
@require_tokenizers
class      UpperCamelCase (				unittest.TestCase ):







		"""simple docstring"""



		@slow
		def       A(  self		:       Union[str, Any]		)    ->   Any:


						'''simple docstring'''
						A	 =      TFFlaubertModel.from_pretrained('jplu/tf-flaubert-small-cased'		)

						A	 =      tf.convert_to_tensor(
						    [[0, 1_5_8, 7_3_5, 2_5_9_2, 1_4_2_4, 6_7_2_7, 8_2, 1]]						,dtype=tf.intaa						,)  # "J'aime flaubert !"

						A	 =      model(_SCREAMING_SNAKE_CASE		)[0]
						A	 =      tf.TensorShape((1, 8, 5_1_2)		)
						self.assertEqual(output.shape						,_SCREAMING_SNAKE_CASE		)
						# compare the actual values for a slice.
						A	 =      tf.convert_to_tensor(
						    [
						        [
						            [-1.8768773, -1.566555, 0.27072418],
						            [-1.6920038, -0.5873505, 1.9329599],
						            [-2.9563985, -1.6993835, 1.7972052],
						        ]
						    ]						,dtype=tf.floataa						,)

						self.assertTrue(np.allclose(output[:, :3, :3].numpy()						,expected_slice.numpy()						,atol=1E-4		)		)













from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin
class      UpperCamelCase (				snake_case__			,       snake_case__			,       snake_case__ ):







		"""simple docstring"""



		snake_case    =   [R"h\.\d+\.attn\.bias", R"h\.\d+\.attn\.masked_bias"]
		@register_to_config
		def __init__(  self		:       Any						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       Optional[int] = None						,_SCREAMING_SNAKE_CASE		:       int = 5_0_2_5_7						,_SCREAMING_SNAKE_CASE		:       int = 1_0_2_4						,_SCREAMING_SNAKE_CASE		:       int = 7_6_8						,_SCREAMING_SNAKE_CASE		:       int = 1_2						,_SCREAMING_SNAKE_CASE		:       int = 1_2						,_SCREAMING_SNAKE_CASE		:       Optional[int] = None						,_SCREAMING_SNAKE_CASE		:       str = "gelu_new"						,_SCREAMING_SNAKE_CASE		:       float = 0.1						,_SCREAMING_SNAKE_CASE		:       float = 0.1						,_SCREAMING_SNAKE_CASE		:       float = 0.1						,_SCREAMING_SNAKE_CASE		:       float = 1E-5						,_SCREAMING_SNAKE_CASE		:       float = 0.02						,_SCREAMING_SNAKE_CASE		:       bool = True						,_SCREAMING_SNAKE_CASE		:       bool = True						,_SCREAMING_SNAKE_CASE		:       bool = False						,_SCREAMING_SNAKE_CASE		:       bool = False						,)    ->   Tuple:


						'''simple docstring'''
						super().__init__()

						A	 =      prefix_length

						if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
										raise ValueError(
										    f'`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'
										    f' `n_embd`: {n_embd} are not equal.'		)

						A	 =      prefix_inner_dim
						A	 =      prefix_hidden_dim

						A	 =      (
						    nn.Linear(self.prefix_inner_dim						,self.prefix_hidden_dim		)
						    if self.prefix_hidden_dim is not None
						    else nn.Identity()
						)
						A	 =      (
						    nn.Linear(self.prefix_hidden_dim						,_SCREAMING_SNAKE_CASE		) if self.prefix_hidden_dim is not None else nn.Identity()
						)

						A	 =      GPTaConfig(
						    vocab_size=_SCREAMING_SNAKE_CASE						,n_positions=_SCREAMING_SNAKE_CASE						,n_embd=_SCREAMING_SNAKE_CASE						,n_layer=_SCREAMING_SNAKE_CASE						,n_head=_SCREAMING_SNAKE_CASE						,n_inner=_SCREAMING_SNAKE_CASE						,activation_function=_SCREAMING_SNAKE_CASE						,resid_pdrop=_SCREAMING_SNAKE_CASE						,embd_pdrop=_SCREAMING_SNAKE_CASE						,attn_pdrop=_SCREAMING_SNAKE_CASE						,layer_norm_epsilon=_SCREAMING_SNAKE_CASE						,initializer_range=_SCREAMING_SNAKE_CASE						,scale_attn_weights=_SCREAMING_SNAKE_CASE						,use_cache=_SCREAMING_SNAKE_CASE						,scale_attn_by_inverse_layer_idx=_SCREAMING_SNAKE_CASE						,reorder_and_upcast_attn=_SCREAMING_SNAKE_CASE						,)
						A	 =      GPTaLMHeadModel(_SCREAMING_SNAKE_CASE		)
		def       A(  self		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       torch.Tensor						,_SCREAMING_SNAKE_CASE		:       torch.Tensor						,_SCREAMING_SNAKE_CASE		:       Optional[torch.Tensor] = None						,_SCREAMING_SNAKE_CASE		:       Optional[torch.Tensor] = None						,)    ->   Tuple:


						'''simple docstring'''
						A	 =      self.transformer.transformer.wte(_SCREAMING_SNAKE_CASE		)
						A	 =      self.encode_prefix(_SCREAMING_SNAKE_CASE		)
						A	 =      self.decode_prefix(_SCREAMING_SNAKE_CASE		)
						A	 =      torch.cat((prefix_embeds, embedding_text)						,dim=1		)

						if labels is not None:
										A	 =      self.get_dummy_token(input_ids.shape[0]						,input_ids.device		)
										A	 =      torch.cat((dummy_token, input_ids)						,dim=1		)
						A	 =      self.transformer(inputs_embeds=_SCREAMING_SNAKE_CASE						,labels=_SCREAMING_SNAKE_CASE						,attention_mask=_SCREAMING_SNAKE_CASE		)
						if self.prefix_hidden_dim is not None:
										return out, hidden
						else:
										return out
		def       A(  self		:       Any						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       torch.device		)    ->   torch.Tensor:


						'''simple docstring'''
						return torch.zeros(_SCREAMING_SNAKE_CASE						,self.prefix_length						,dtype=torch.intaa						,device=_SCREAMING_SNAKE_CASE		)
		def       A(  self		:       List[str]						,_SCREAMING_SNAKE_CASE		:       int		)    ->   Optional[Any]:


						'''simple docstring'''
						return self.encode_prefix(_SCREAMING_SNAKE_CASE		)
		@torch.no_grad()
		def       A(  self		:       List[str]						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       int						,_SCREAMING_SNAKE_CASE		:       Union[str, Any]		)    ->   str:


						'''simple docstring'''
						A	 =      torch.split(_SCREAMING_SNAKE_CASE						,1						,dim=0		)
						A	 =      []
						A	 =      []
						for feature in features:
										A	 =      self.decode_prefix(feature.to(_SCREAMING_SNAKE_CASE		)		)  # back to the clip feature
										# Only support beam search for now
										A ,		A	 =      self.generate_beam(
										    input_embeds=_SCREAMING_SNAKE_CASE						,device=_SCREAMING_SNAKE_CASE						,eos_token_id=_SCREAMING_SNAKE_CASE		)
										generated_tokens.append(output_tokens[0]		)
										generated_seq_lengths.append(seq_lengths[0]		)
						A	 =      torch.stack(_SCREAMING_SNAKE_CASE		)
						A	 =      torch.stack(_SCREAMING_SNAKE_CASE		)
						return generated_tokens, generated_seq_lengths







		@torch.no_grad()
		def       A(  self		:       Dict						,_SCREAMING_SNAKE_CASE		:       List[str]=None						,_SCREAMING_SNAKE_CASE		:       str=None						,_SCREAMING_SNAKE_CASE		:       Any=None						,_SCREAMING_SNAKE_CASE		:       int = 5						,_SCREAMING_SNAKE_CASE		:       int = 6_7						,_SCREAMING_SNAKE_CASE		:       float = 1.0						,_SCREAMING_SNAKE_CASE		:       Optional[int] = None						,)    ->   str:


						'''simple docstring'''
						A	 =      eos_token_id
						A	 =      None
						A	 =      None
						A	 =      torch.ones(_SCREAMING_SNAKE_CASE						,device=_SCREAMING_SNAKE_CASE						,dtype=torch.int		)
						A	 =      torch.zeros(_SCREAMING_SNAKE_CASE						,device=_SCREAMING_SNAKE_CASE						,dtype=torch.bool		)

						if input_embeds is not None:
										A	 =      input_embeds
						else:
										A	 =      self.transformer.transformer.wte(_SCREAMING_SNAKE_CASE		)

						for i in range(_SCREAMING_SNAKE_CASE		):
										A	 =      self.transformer(inputs_embeds=_SCREAMING_SNAKE_CASE		)
										A	 =      outputs.logits
										A	 =      logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
										A	 =      logits.softmax(-1		).log()

										if scores is None:
														A ,		A	 =      logits.topk(_SCREAMING_SNAKE_CASE						,-1		)
														A	 =      generated.expand(_SCREAMING_SNAKE_CASE						,*generated.shape[1:]		)
														A ,		A	 =      next_tokens.permute(1						,0		), scores.squeeze(0		)
														if tokens is None:
																		A	 =      next_tokens
														else:
																		A	 =      tokens.expand(_SCREAMING_SNAKE_CASE						,*tokens.shape[1:]		)
																		A	 =      torch.cat((tokens, next_tokens)						,dim=1		)
										else:
														A	 =      -float(np.inf		)
														A	 =      0
														A	 =      scores[:, None] + logits
														seq_lengths[~is_stopped] += 1
														A	 =      scores_sum / seq_lengths[:, None]
														A ,		A	 =      scores_sum_average.view(-1		).topk(_SCREAMING_SNAKE_CASE						,-1		)
														A	 =      next_tokens // scores_sum.shape[1]
														A	 =      seq_lengths[next_tokens_source]
														A	 =      next_tokens % scores_sum.shape[1]
														A	 =      next_tokens.unsqueeze(1		)
														A	 =      tokens[next_tokens_source]
														A	 =      torch.cat((tokens, next_tokens)						,dim=1		)
														A	 =      generated[next_tokens_source]
														A	 =      scores_sum_average * seq_lengths
														A	 =      is_stopped[next_tokens_source]

										A	 =      self.transformer.transformer.wte(next_tokens.squeeze()		).view(generated.shape[0]						,1						,-1		)
										A	 =      torch.cat((generated, next_token_embed)						,dim=1		)
										A	 =      is_stopped + next_tokens.eq(_SCREAMING_SNAKE_CASE		).squeeze()
										if is_stopped.all():
														break

						A	 =      scores / seq_lengths
						A	 =      scores.argsort(descending=_SCREAMING_SNAKE_CASE		)
						# tokens tensors are already padded to max_seq_length
						A	 =      [tokens[i] for i in order]
						A	 =      torch.stack(_SCREAMING_SNAKE_CASE						,dim=0		)
						A	 =      torch.tensor([seq_lengths[i] for i in order]						,dtype=seq_lengths.dtype		)
						return output_texts, seq_lengths





