class_docstr: A class containing all of the functions supporting generation, to be used as a mixin in
  :class:`~transformers.PreTrainedModel`.
class_docstr_tok: ["A","class","containing","all","of","the","functions","supporting","generation",",","to","be","used","as","a","mixin","in",":","class",":","`","~transformers.PreTrainedModel","`","."]
docstr: Generates sequences for models with a language modeling head using multinomial sampling.
  
  Parameters:
  
      input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`):
          The sequence used as a prompt for the generation.
      logits_processor (:obj:`LogitsProcessorList`, `optional`):
          An instance of :class:`~transformers.LogitsProcessorList`. List of instances of class derived from
          :class:`~transformers.LogitsProcessor` used to modify the prediction scores of the language modeling
          head applied at each generation step.
      stopping_criteria (:obj:`StoppingCriteriaList`, `optional`):
          An instance of :class:`~transformers.StoppingCriteriaList`. List of instances of class derived from
          :class:`~transformers.StoppingCriteria` used to tell if the generation loop should stop.
      logits_warper (:obj:`LogitsProcessorList`, `optional`):
          An instance of :class:`~transformers.LogitsProcessorList`. List of instances of class derived from
          :class:`~transformers.LogitsWarper` used to warp the prediction score distribution of the language
          modeling head applied before multinomial sampling at each generation step.
      max_length (:obj:`int`, `optional`, defaults to 20):
          **DEPRECATED**. Use :obj:`logits_processor` or :obj:`stopping_criteria` directly to cap the number of
          generated tokens. The maximum length of the sequence to be generated.
      pad_token_id (:obj:`int`, `optional`):
          The id of the `padding` token.
      eos_token_id (:obj:`int`, `optional`):
          The id of the `end-of-sequence` token.
      output_attentions (:obj:`bool`, `optional`, defaults to `False`):
          Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under
          returned tensors for more details.
      output_hidden_states (:obj:`bool`, `optional`, defaults to `False`):
          Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors
          for more details.
      output_scores (:obj:`bool`, `optional`, defaults to `False`):
          Whether or not to return the prediction scores. See ``scores`` under returned tensors for more details.
      return_dict_in_generate (:obj:`bool`, `optional`, defaults to `False`):
          Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple.
      synced_gpus (:obj:`bool`, `optional`, defaults to :obj:`False`):
          Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
      model_kwargs:
          Additional model specific kwargs will be forwarded to the :obj:`forward` function of the model. If
          model is an encoder-decoder model the kwargs should include :obj:`encoder_outputs`.
  
  Return:
      :class:`~transformers.generation_utils.SampleDecoderOnlyOutput`,
      :class:`~transformers.generation_utils.SampleEncoderDecoderOutput` or obj:`torch.LongTensor`: A
      :obj:`torch.LongTensor` containing the generated tokens (default behaviour) or a
      :class:`~transformers.generation_utils.SampleDecoderOnlyOutput` if
      ``model.config.is_encoder_decoder=False`` and ``return_dict_in_generate=True`` or a
      :class:`~transformers.generation_utils.SampleEncoderDecoderOutput` if
      ``model.config.is_encoder_decoder=True``.
  
  Examples::
  
      >>> from transformers import (
      ...    AutoTokenizer,
      ...    AutoModelForCausalLM,
      ...    LogitsProcessorList,
      ...    MinLengthLogitsProcessor,
      ...    TopKLogitsWarper,
      ...    TemperatureLogitsWarper,
      ... )
  
      >>> tokenizer = AutoTokenizer.from_pretrained("gpt2")
      >>> model = AutoModelForCausalLM.from_pretrained("gpt2")
  
      >>> # set pad_token_id to eos_token_id because GPT2 does not have a EOS token
      >>> model.config.pad_token_id = model.config.eos_token_id
  
      >>> input_prompt = "Today is a beautiful day, and"
      >>> input_ids = tokenizer(input_prompt, return_tensors="pt").input_ids
  
      >>> # instantiate logits processors
      >>> logits_processor = LogitsProcessorList([
      ...     MinLengthLogitsProcessor(15, eos_token_id=model.config.eos_token_id),
      ... ])
      >>> # instantiate logits processors
      >>> logits_warper = LogitsProcessorList([
      ...     TopKLogitsWarper(50),
      ...     TemperatureLogitsWarper(0.7),
      ... ])
  
      >>> outputs = model.sample(input_ids, logits_processor=logits_processor, logits_warper=logits_warper)
  
      >>> print("Generated:", tokenizer.batch_decode(outputs, skip_special_tokens=True))
docstr_tok: ["Generates","sequences","for","models","with","a","language","modeling","head","using","multinomial","sampling",".","Parameters",":","input_ids","(",":","obj",":","`","torch.LongTensor","`","of","shape",":","obj",":","`","(","batch_size",",","sequence_length",")","`",")",":","The","sequence","used","as","a","prompt","for","the","generation",".","logits_processor","(",":","obj",":","`","LogitsProcessorList","`",",","`","optional","`",")",":","An","instance","of",":","class",":","`","~transformers.LogitsProcessorList","`",".","List","of","instances","of","class","derived","from",":","class",":","`","~transformers.LogitsProcessor","`","used","to","modify","the","prediction","scores","of","the","language","modeling","head","applied","at","each","generation","step",".","stopping_criteria","(",":","obj",":","`","StoppingCriteriaList","`",",","`","optional","`",")",":","An","instance","of",":","class",":","`","~transformers.StoppingCriteriaList","`",".","List","of","instances","of","class","derived","from",":","class",":","`","~transformers.StoppingCriteria","`","used","to","tell","if","the","generation","loop","should","stop",".","logits_warper","(",":","obj",":","`","LogitsProcessorList","`",",","`","optional","`",")",":","An","instance","of",":","class",":","`","~transformers.LogitsProcessorList","`",".","List","of","instances","of","class","derived","from",":","class",":","`","~transformers.LogitsWarper","`","used","to","warp","the","prediction","score","distribution","of","the","language","modeling","head","applied","before","multinomial","sampling","at","each","generation","step",".","max_length","(",":","obj",":","`","int","`",",","`","optional","`",",","defaults","to","20",")",":","*","*","DEPRECATED","*","*",".","Use",":","obj",":","`","logits_processor","`","or",":","obj",":","`","stopping_criteria","`","directly","to","cap","the","number","of","generated","tokens",".","The","maximum","length","of","the","sequence","to","be","generated",".","pad_token_id","(",":","obj",":","`","int","`",",","`","optional","`",")",":","The","id","of","the","`","padding","`","token",".","eos_token_id","(",":","obj",":","`","int","`",",","`","optional","`",")",":","The","id","of","the","`","end-of-sequence","`","token",".","output_attentions","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to","`","False","`",")",":","Whether","or","not","to","return","the","attentions","tensors","of","all","attention","layers",".","See","``","attentions","``","under","returned","tensors","for","more","details",".","output_hidden_states","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to","`","False","`",")",":","Whether","or","not","to","return","the","hidden","states","of","all","layers",".","See","``","hidden_states","``","under","returned","tensors","for","more","details",".","output_scores","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to","`","False","`",")",":","Whether","or","not","to","return","the","prediction","scores",".","See","``","scores","``","under","returned","tensors","for","more","details",".","return_dict_in_generate","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to","`","False","`",")",":","Whether","or","not","to","return","a",":","class",":","`","~transformers.file_utils.ModelOutput","`","instead","of","a","plain","tuple",".","synced_gpus","(",":","obj",":","`","bool","`",",","`","optional","`",",","defaults","to",":","obj",":","`","False","`",")",":","Whether","to","continue","running","the","while","loop","until","max_length","(","needed","for","ZeRO","stage","3",")","model_kwargs",":","Additional","model","specific","kwargs","will","be","forwarded","to","the",":","obj",":","`","forward","`","function","of","the","model",".","If","model","is","an","encoder-decoder","model","the","kwargs","should","include",":","obj",":","`","encoder_outputs","`",".","Return",":",":","class",":","`","~transformers.generation_utils.SampleDecoderOnlyOutput","`",",",":","class",":","`","~transformers.generation_utils.SampleEncoderDecoderOutput","`","or","obj",":","`","torch.LongTensor","`",":","A",":","obj",":","`","torch.LongTensor","`","containing","the","generated","tokens","(","default","behaviour",")","or","a",":","class",":","`","~transformers.generation_utils.SampleDecoderOnlyOutput","`","if","``","model.config.is_encoder_decoder=False","``","and","``","return_dict_in_generate=True","``","or","a",":","class",":","`","~transformers.generation_utils.SampleEncoderDecoderOutput","`","if","``","model.config.is_encoder_decoder=True","``",".","Examples",":",":",">",">",">","from","transformers","import","(","...","AutoTokenizer",",","...","AutoModelForCausalLM",",","...","LogitsProcessorList",",","...","MinLengthLogitsProcessor",",","...","TopKLogitsWarper",",","...","TemperatureLogitsWarper",",","...",")",">",">",">","tokenizer","=","AutoTokenizer.from_pretrained","(","``","gpt2","''",")",">",">",">","model","=","AutoModelForCausalLM.from_pretrained","(","``","gpt2","''",")",">",">",">","#","set","pad_token_id","to","eos_token_id","because","GPT2","does","not","have","a","EOS","token",">",">",">","model.config.pad_token_id","=","model.config.eos_token_id",">",">",">","input_prompt","=","``","Today","is","a","beautiful","day",",","and","''",">",">",">","input_ids","=","tokenizer","(","input_prompt",",","return_tensors=","''","pt","''",")",".input_ids",">",">",">","#","instantiate","logits","processors",">",">",">","logits_processor","=","LogitsProcessorList","(","[","...","MinLengthLogitsProcessor","(","15",",","eos_token_id=model.config.eos_token_id",")",",","...","]",")",">",">",">","#","instantiate","logits","processors",">",">",">","logits_warper","=","LogitsProcessorList","(","[","...","TopKLogitsWarper","(","50",")",",","...","TemperatureLogitsWarper","(","0.7",")",",","...","]",")",">",">",">","outputs","=","model.sample","(","input_ids",",","logits_processor=logits_processor",",","logits_warper=logits_warper",")",">",">",">","print","(","``","Generated",":","''",",","tokenizer.batch_decode","(","outputs",",","skip_special_tokens=True",")",")"]
code: def sample(
      self,
      input_ids: "torch.LongTensor",
      logits_processor: Optional[LogitsProcessorList] = None,
      stopping_criteria: Optional[StoppingCriteriaList] = None,
      logits_warper: Optional[LogitsProcessorList] = None,
      max_length: Optional[int] = None,
      pad_token_id: Optional[int] = None,
      eos_token_id: Optional[int] = None,
      output_attentions: Optional[bool] = None,
      output_hidden_states: Optional[bool] = None,
      output_scores: Optional[bool] = None,
      return_dict_in_generate: Optional[bool] = None,
      synced_gpus: Optional[bool] = None,
      **model_kwargs,
  ) -> Union[SampleOutput, "torch.LongTensor"]:
      r"""
      Generates sequences for models with a language modeling head using multinomial sampling.
  
      Parameters:
  
          input_ids (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`):
              The sequence used as a prompt for the generation.
          logits_processor (:obj:`LogitsProcessorList`, `optional`):
              An instance of :class:`~transformers.LogitsProcessorList`. List of instances of class derived from
              :class:`~transformers.LogitsProcessor` used to modify the prediction scores of the language modeling
              head applied at each generation step.
          stopping_criteria (:obj:`StoppingCriteriaList`, `optional`):
              An instance of :class:`~transformers.StoppingCriteriaList`. List of instances of class derived from
              :class:`~transformers.StoppingCriteria` used to tell if the generation loop should stop.
          logits_warper (:obj:`LogitsProcessorList`, `optional`):
              An instance of :class:`~transformers.LogitsProcessorList`. List of instances of class derived from
              :class:`~transformers.LogitsWarper` used to warp the prediction score distribution of the language
              modeling head applied before multinomial sampling at each generation step.
          max_length (:obj:`int`, `optional`, defaults to 20):
              **DEPRECATED**. Use :obj:`logits_processor` or :obj:`stopping_criteria` directly to cap the number of
              generated tokens. The maximum length of the sequence to be generated.
          pad_token_id (:obj:`int`, `optional`):
              The id of the `padding` token.
          eos_token_id (:obj:`int`, `optional`):
              The id of the `end-of-sequence` token.
          output_attentions (:obj:`bool`, `optional`, defaults to `False`):
              Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under
              returned tensors for more details.
          output_hidden_states (:obj:`bool`, `optional`, defaults to `False`):
              Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors
              for more details.
          output_scores (:obj:`bool`, `optional`, defaults to `False`):
              Whether or not to return the prediction scores. See ``scores`` under returned tensors for more details.
          return_dict_in_generate (:obj:`bool`, `optional`, defaults to `False`):
              Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple.
          synced_gpus (:obj:`bool`, `optional`, defaults to :obj:`False`):
              Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
          model_kwargs:
              Additional model specific kwargs will be forwarded to the :obj:`forward` function of the model. If
              model is an encoder-decoder model the kwargs should include :obj:`encoder_outputs`.
  
      Return:
          :class:`~transformers.generation_utils.SampleDecoderOnlyOutput`,
          :class:`~transformers.generation_utils.SampleEncoderDecoderOutput` or obj:`torch.LongTensor`: A
          :obj:`torch.LongTensor` containing the generated tokens (default behaviour) or a
          :class:`~transformers.generation_utils.SampleDecoderOnlyOutput` if
          ``model.config.is_encoder_decoder=False`` and ``return_dict_in_generate=True`` or a
          :class:`~transformers.generation_utils.SampleEncoderDecoderOutput` if
          ``model.config.is_encoder_decoder=True``.
  
      Examples::
  
          >>> from transformers import (
          ...    AutoTokenizer,
          ...    AutoModelForCausalLM,
          ...    LogitsProcessorList,
          ...    MinLengthLogitsProcessor,
          ...    TopKLogitsWarper,
          ...    TemperatureLogitsWarper,
          ... )
  
          >>> tokenizer = AutoTokenizer.from_pretrained("gpt2")
          >>> model = AutoModelForCausalLM.from_pretrained("gpt2")
  
          >>> # set pad_token_id to eos_token_id because GPT2 does not have a EOS token
          >>> model.config.pad_token_id = model.config.eos_token_id
  
          >>> input_prompt = "Today is a beautiful day, and"
          >>> input_ids = tokenizer(input_prompt, return_tensors="pt").input_ids
  
          >>> # instantiate logits processors
          >>> logits_processor = LogitsProcessorList([
          ...     MinLengthLogitsProcessor(15, eos_token_id=model.config.eos_token_id),
          ... ])
          >>> # instantiate logits processors
          >>> logits_warper = LogitsProcessorList([
          ...     TopKLogitsWarper(50),
          ...     TemperatureLogitsWarper(0.7),
          ... ])
  
          >>> outputs = model.sample(input_ids, logits_processor=logits_processor, logits_warper=logits_warper)
  
          >>> print("Generated:", tokenizer.batch_decode(outputs, skip_special_tokens=True))
      """
  
      # init values
      logits_processor = (
          logits_processor
          if logits_processor is not None
          else LogitsProcessorList()
      )
      stopping_criteria = (
          stopping_criteria
          if stopping_criteria is not None
          else StoppingCriteriaList()
      )
      if max_length is not None:
          logger.warning(
              "`max_length` is deprecated in this function, use `stopping_criteria=StoppingCriteriaList(MaxLengthCriteria(max_length=max_length))` instead.",
              UserWarning,
          )
          stopping_criteria = validate_stopping_criteria(
              stopping_criteria, max_length
          )
      logits_warper = (
          logits_warper
          if logits_warper is not None
          else LogitsProcessorList()
      )
      pad_token_id = (
          pad_token_id
          if pad_token_id is not None
          else self.config.pad_token_id
      )
      eos_token_id = (
          eos_token_id
          if eos_token_id is not None
          else self.config.eos_token_id
      )
      output_scores = (
          output_scores
          if output_scores is not None
          else self.config.output_scores
      )
      output_attentions = (
          output_attentions
          if output_attentions is not None
          else self.config.output_attentions
      )
      output_hidden_states = (
          output_hidden_states
          if output_hidden_states is not None
          else self.config.output_hidden_states
      )
      return_dict_in_generate = (
          return_dict_in_generate
          if return_dict_in_generate is not None
          else self.config.return_dict_in_generate
      )
  
      # init attention / hidden states / scores tuples
      scores = (
          () if (return_dict_in_generate and output_scores) else None
      )
      decoder_attentions = (
          ()
          if (return_dict_in_generate and output_attentions)
          else None
      )
      cross_attentions = (
          ()
          if (return_dict_in_generate and output_attentions)
          else None
      )
      decoder_hidden_states = (
          ()
          if (return_dict_in_generate and output_hidden_states)
          else None
      )
  
      # if model is an encoder-decoder, retrieve encoder attention weights and hidden states
      if return_dict_in_generate and self.config.is_encoder_decoder:
          encoder_attentions = (
              model_kwargs["encoder_outputs"].get("attentions")
              if output_attentions
              else None
          )
          encoder_hidden_states = (
              model_kwargs["encoder_outputs"].get("hidden_states")
              if output_hidden_states
              else None
          )
  
      # keep track of which sequences are already finished
      unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1)
      cur_len = input_ids.shape[-1]
  
      this_peer_finished = False  # used by synced_gpus only
      # auto-regressive generation
      while True:
          if synced_gpus:
              # Under synced_gpus the `forward` call must continue until all gpus complete their sequence.
              # The following logic allows an early break if all peers finished generating their sequence
              this_peer_finished_flag = torch.tensor(
                  0.0 if this_peer_finished else 1.0
              ).to(input_ids.device)
              # send 0.0 if we finished, 1.0 otherwise
              dist.all_reduce(
                  this_peer_finished_flag, op=dist.ReduceOp.SUM
              )
              # did all peers finish? the reduced sum will be 0.0 then
              if this_peer_finished_flag.item() == 0.0:
                  break
  
          # prepare model inputs
          model_inputs = self.prepare_inputs_for_generation(
              input_ids, **model_kwargs
          )
  
          # forward pass to get next token
          outputs = self(
              **model_inputs,
              return_dict=True,
              output_attentions=output_attentions,
              output_hidden_states=output_hidden_states,
          )
  
          if synced_gpus and this_peer_finished:
              cur_len = cur_len + 1
              continue  # don't waste resources running the code we don't need
  
          next_token_logits = outputs.logits[:, -1, :]
  
          # pre-process distribution
          next_token_scores = logits_processor(
              input_ids, next_token_logits
          )
          next_token_scores = logits_warper(
              input_ids, next_token_scores
          )
  
          # Store scores, attentions and hidden_states when required
          if return_dict_in_generate:
              if output_scores:
                  scores += (next_token_scores,)
              if output_attentions:
                  decoder_attentions += (
                      (outputs.decoder_attentions,)
                      if self.config.is_encoder_decoder
                      else (outputs.attentions,)
                  )
                  if self.config.is_encoder_decoder:
                      cross_attentions += (outputs.cross_attentions,)
  
              if output_hidden_states:
                  decoder_hidden_states += (
                      (outputs.decoder_hidden_states,)
                      if self.config.is_encoder_decoder
                      else (outputs.hidden_states,)
                  )
  
          # sample
          probs = nn.functional.softmax(next_token_scores, dim=-1)
          next_tokens = torch.multinomial(probs, num_samples=1).squeeze(
              1
          )
  
          # finished sentences should have their next token be a padding token
          if eos_token_id is not None:
              assert (
                  pad_token_id is not None
              ), "If eos_token_id is defined, make sure that pad_token_id is defined."
              next_tokens = (
                  next_tokens * unfinished_sequences
                  + pad_token_id * (1 - unfinished_sequences)
              )
  
          # update generated ids, model inputs, and length for next step
          input_ids = torch.cat(
              [input_ids, next_tokens[:, None]], dim=-1
          )
          model_kwargs = self._update_model_kwargs_for_generation(
              outputs,
              model_kwargs,
              is_encoder_decoder=self.config.is_encoder_decoder,
          )
          cur_len = cur_len + 1
  
          # if eos_token was found in one sentence, set sentence to finished
          if eos_token_id is not None:
              unfinished_sequences = unfinished_sequences.mul(
                  (next_tokens != eos_token_id).long()
              )
  
          # stop when each sentence is finished, or if we exceed the maximum length
          if unfinished_sequences.max() == 0 or stopping_criteria(
              input_ids, scores
          ):
              if not synced_gpus:
                  break
              else:
                  this_peer_finished = True
  
      if return_dict_in_generate:
          if self.config.is_encoder_decoder:
              return SampleEncoderDecoderOutput(
                  sequences=input_ids,
                  scores=scores,
                  encoder_attentions=encoder_attentions,
                  encoder_hidden_states=encoder_hidden_states,
                  decoder_attentions=decoder_attentions,
                  cross_attentions=cross_attentions,
                  decoder_hidden_states=decoder_hidden_states,
              )
          else:
              return SampleDecoderOnlyOutput(
                  sequences=input_ids,
                  scores=scores,
                  attentions=decoder_attentions,
                  hidden_states=decoder_hidden_states,
              )
      else:
          return input_ids
  
imports: from dataclasses import dataclass
  from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
  from .file_utils import ModelOutput
  from .generation_beam_search import BeamScorer, BeamSearchScorer
  from .generation_logits_process import EncoderNoRepeatNGramLogitsProcessor, ForcedBOSTokenLogitsProcessor, ForcedEOSTokenLogitsProcessor, HammingDiversityLogitsProcessor, InfNanRemoveLogitsProcessor, LogitsProcessorList, MinLengthLogitsProcessor, NoBadWordsLogitsProcessor, NoRepeatNGramLogitsProcessor, PrefixConstrainedLogitsProcessor, RepetitionPenaltyLogitsProcessor, TemperatureLogitsWarper, TopKLogitsWarper, TopPLogitsWarper
  from .generation_stopping_criteria import MaxLengthCriteria, MaxNewTokensCriteria, MaxTimeCriteria, StoppingCriteriaList, validate_stopping_criteria
  from fastNLP.envs.imports import _NEED_IMPORT_TORCH
  from fastNLP.core.log import logger
label_desc: Use image node_id 17 for calling the GenerationMixin obj's underlying member method code with example usage: obj.sample(input_ids, logits_processor, stopping_criteria, logits_warper, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus) and returns: input_ids, SampleEncoderDecoderOutput, SampleDecoderOnlyOutput