docstr: Generate text sequences based on provided prompts using the language generation model.
  
  Args:
      prompt_tokens (List[List[int]]): List of tokenized prompts, where each prompt is represented as a list of integers.
      max_gen_len (int): Maximum length of the generated text sequence.
      temperature (float, optional): Temperature value for controlling randomness in sampling. Defaults to 0.6.
      top_p (float, optional): Top-p probability threshold for nucleus sampling. Defaults to 0.9.
      logprobs (bool, optional): Flag indicating whether to compute token log probabilities. Defaults to False.
      echo (bool, optional): Flag indicating whether to include prompt tokens in the generated output. Defaults to False.
  
  Returns:
      Tuple[List[List[int]], Optional[List[List[float]]]]: A tuple containing generated token sequences and, if logprobs is True, corresponding token log probabilities.
  
  Note:
      This method uses the provided prompts as a basis for generating text. It employs nucleus sampling to produce text with controlled randomness.
      If logprobs is True, token log probabilities are computed for each generated token.
docstr_tok: ["Generate","text","sequences","based","on","provided","prompts","using","the","language","generation","model",".","Args",":","prompt_tokens","(","List","[","List","[","int","]","]",")",":","List","of","tokenized","prompts",",","where","each","prompt","is","represented","as","a","list","of","integers",".","max_gen_len","(","int",")",":","Maximum","length","of","the","generated","text","sequence",".","temperature","(","float",",","optional",")",":","Temperature","value","for","controlling","randomness","in","sampling",".","Defaults","to","0.6.","top_p","(","float",",","optional",")",":","Top-p","probability","threshold","for","nucleus","sampling",".","Defaults","to","0.9.","logprobs","(","bool",",","optional",")",":","Flag","indicating","whether","to","compute","token","log","probabilities",".","Defaults","to","False",".","echo","(","bool",",","optional",")",":","Flag","indicating","whether","to","include","prompt","tokens","in","the","generated","output",".","Defaults","to","False",".","Returns",":","Tuple","[","List","[","List","[","int","]","]",",","Optional","[","List","[","List","[","float","]","]","]","]",":","A","tuple","containing","generated","token","sequences","and",",","if","logprobs","is","True",",","corresponding","token","log","probabilities",".","Note",":","This","method","uses","the","provided","prompts","as","a","basis","for","generating","text",".","It","employs","nucleus","sampling","to","produce","text","with","controlled","randomness",".","If","logprobs","is","True",",","token","log","probabilities","are","computed","for","each","generated","token","."]
code: def generate(
      self,
      prompt_tokens: List[List[int]],
      max_gen_len: int,
      temperature: float = 0.6,
      top_p: float = 0.9,
      logprobs: bool = False,
      echo: bool = False,
  ) -> Tuple[List[List[int]], Optional[List[List[float]]]]:
      """
      Generate text sequences based on provided prompts using the language generation model.
  
      Args:
          prompt_tokens (List[List[int]]): List of tokenized prompts, where each prompt is represented as a list of integers.
          max_gen_len (int): Maximum length of the generated text sequence.
          temperature (float, optional): Temperature value for controlling randomness in sampling. Defaults to 0.6.
          top_p (float, optional): Top-p probability threshold for nucleus sampling. Defaults to 0.9.
          logprobs (bool, optional): Flag indicating whether to compute token log probabilities. Defaults to False.
          echo (bool, optional): Flag indicating whether to include prompt tokens in the generated output. Defaults to False.
  
      Returns:
          Tuple[List[List[int]], Optional[List[List[float]]]]: A tuple containing generated token sequences and, if logprobs is True, corresponding token log probabilities.
  
      Note:
          This method uses the provided prompts as a basis for generating text. It employs nucleus sampling to produce text with controlled randomness.
          If logprobs is True, token log probabilities are computed for each generated token.
  
      """
      params = self.model.params
      bsz = len(prompt_tokens)
      assert bsz <= params.max_batch_size, (bsz, params.max_batch_size)
  
      min_prompt_len = min(len(t) for t in prompt_tokens)
      max_prompt_len = max(len(t) for t in prompt_tokens)
      assert max_prompt_len <= params.max_seq_len
      total_len = min(params.max_seq_len, max_gen_len + max_prompt_len)
  
      pad_id = self.tokenizer.pad_id
      tokens = torch.full(
          (bsz, total_len), pad_id, dtype=torch.long, device="cuda"
      )
      for k, t in enumerate(prompt_tokens):
          tokens[k, : len(t)] = torch.tensor(
              t, dtype=torch.long, device="cuda"
          )
      if logprobs:
          token_logprobs = torch.zeros_like(tokens, dtype=torch.float)
  
      prev_pos = 0
      eos_reached = torch.tensor([False] * bsz, device="cuda")
      input_text_mask = tokens != pad_id
      if min_prompt_len == total_len:
          logits = self.model.forward(tokens, prev_pos)
          token_logprobs = -F.cross_entropy(
              input=logits.transpose(1, 2),
              target=tokens,
              reduction="none",
              ignore_index=pad_id,
          )
  
      for cur_pos in range(min_prompt_len, total_len):
          logits = self.model.forward(
              tokens[:, prev_pos:cur_pos], prev_pos
          )
          if temperature > 0:
              probs = torch.softmax(logits[:, -1] / temperature, dim=-1)
              next_token = sample_top_p(probs, top_p)
          else:
              next_token = torch.argmax(logits[:, -1], dim=-1)
  
          next_token = next_token.reshape(-1)
          # only replace token if prompt has already been generated
          next_token = torch.where(
              input_text_mask[:, cur_pos],
              tokens[:, cur_pos],
              next_token,
          )
          tokens[:, cur_pos] = next_token
          if logprobs:
              token_logprobs[
                  :, prev_pos + 1 : cur_pos + 1
              ] = -F.cross_entropy(
                  input=logits.transpose(1, 2),
                  target=tokens[:, prev_pos + 1 : cur_pos + 1],
                  reduction="none",
                  ignore_index=pad_id,
              )
          eos_reached |= (~input_text_mask[:, cur_pos]) & (
              next_token == self.tokenizer.eos_id
          )
          prev_pos = cur_pos
          if all(eos_reached):
              break
  
      if logprobs:
          token_logprobs = token_logprobs.tolist()
      out_tokens, out_logprobs = [], []
      for i, toks in enumerate(tokens.tolist()):
          # cut to max gen len
          start = 0 if echo else len(prompt_tokens[i])
          toks = toks[start : len(prompt_tokens[i]) + max_gen_len]
          probs = None
          if logprobs:
              probs = token_logprobs[i][
                  start : len(prompt_tokens[i]) + max_gen_len
              ]
          # cut to eos tok if any
          if self.tokenizer.eos_id in toks:
              eos_idx = toks.index(self.tokenizer.eos_id)
              toks = toks[:eos_idx]
              probs = probs[:eos_idx] if logprobs else None
          out_tokens.append(toks)
          out_logprobs.append(probs)
      return (out_tokens, out_logprobs if logprobs else None)
  
file_path: mixtral-kit/mixtralkit/mixtral/generation.py
imports: import json
  import os
  import sys
  import time
  from pathlib import Path
  from typing import List, Optional, Tuple, TypedDict
  import torch
  import torch.nn.functional
  from mixtralkit.layers import Tokenizer, MoETorchTransformer, MixtralModelArgs
  from mixtralkit.utils import sample_top_p
  from mixtralkit.utils.generation import CompletionPrediction, Dialog, ChatPrediction
label_desc: Use image node_id 3 for calling the Mixtral obj's underlying member method code with example usage: obj.generate(prompt_tokens, max_gen_len, temperature, top_p, logprobs, echo) and returns: out_tokens,  unknown