docstr: Generate results given a list of inputs and get logits of the first new token over choices.
  
  Args:
      inputs: A list of strings.
      max_new_tokens: Max new tokens for generation.
      kwargs: Key arguments for generation
  
  Returns:
      A list of generated strings and logits over choices.
  
  Note:
      Currently the function only returns the logits of the first new token.
      It is used for single choice question.
      For multiple choices question, please avoid using the loss over choices.
      You should set argument choices as None in self.inference().
docstr_tok: ["Generate","results","given","a","list","of","inputs","and","get","logits","of","the","first","new","token","over","choices",".","Args",":","inputs",":","A","list","of","strings",".","max_new_tokens",":","Max","new","tokens","for","generation",".","kwargs",":","Key","arguments","for","generation","Returns",":","A","list","of","generated","strings","and","logits","over","choices",".","Note",":","Currently","the","function","only","returns","the","logits","of","the","first","new","token",".","It","is","used","for","single","choice","question",".","For","multiple","choices","question",",","please","avoid","using","the","loss","over","choices",".","You","should","set","argument","choices","as","None","in","self.inference","(",")","."]
code: def generate(
      self, inputs: List[str], max_new_tokens: int, **kwargs
  ) -> List[str]:
      """Generate results given a list of inputs and get logits of the first new token over choices.
  
      Args:
          inputs: A list of strings.
          max_new_tokens: Max new tokens for generation.
          kwargs: Key arguments for generation
  
      Returns:
          A list of generated strings and logits over choices.
  
      Note:
          Currently the function only returns the logits of the first new token.
          It is used for single choice question.
          For multiple choices question, please avoid using the loss over choices.
          You should set argument choices as None in self.inference().
  
      """
      # Follow the process of model.chat() method in modeling_chatglm2.py
      # See https://huggingface.co/THUDM/chatglm2-6b/blob/main/modeling_chatglm.py#L1020
      # See https://huggingface.co/THUDM/chatglm2-6b/blob/main/modeling_chatglm.py#L1001
  
      query = []
      for input in inputs:
          prompt = self.tokenizer.build_prompt(input, None)
          query.append(prompt)
  
      truncated_query = self._get_truncated_prompts(
          query, max_new_tokens
      )
  
      encoded_inputs = self.tokenizer(
          truncated_query,
          padding=True,
          truncation=True,
          return_tensors="pt",
          max_length=self.model_max_length - max_new_tokens,
      ).to(torch.cuda.current_device())
  
      # Set output_scores=True to get prediction scores.
      outputs = self.model.generate(
          **encoded_inputs,
          max_new_tokens=max_new_tokens,
          return_dict_in_generate=True,
          output_scores=True,
          **kwargs,
      )
  
      # We only need to decode predicted tokens.
      sequences = outputs.sequences[
          :, encoded_inputs["input_ids"].shape[1] :
      ]
  
      scores = []
      if self.indices_for_choices:
          # If the question is a single-choice question, we will return the scores of specific indices for first predicted token.
          # The indices are the tokenization results of the options for the single-choice question.
          # For example, if the options of the question are A, B, C and D, we only returns scores at indices of A, B, C and D.
          for option_indices in self.indices_for_choices:
              scores.append(
                  outputs.scores[0][:, option_indices].detach().cpu()
              )
  
          scores = torch.max(torch.stack(scores), dim=0)[0]
  
      decoded_sequences = self.tokenizer.batch_decode(
          sequences, skip_special_tokens=True
      )
  
      return decoded_sequences, scores
  
file_path: colossalai/applications/ColossalEval/colossal_eval/models/chatglm.py
imports: import copy
  from typing import List
  import torch
  from .huggingface import HuggingFaceModel
label_desc: Use image node_id 2 for calling the ChatGLM2Model obj's underlying member method code with example usage: obj.generate(inputs, max_new_tokens) and returns: decoded_sequences,  scores