code: def _tokenize_prompts_and_batch(prompts, tokens_to_generate, add_BOS):
      """Given a set of prompts and number of tokens to generate:
      - tokenize prompts
      - set the sequence length to be the max of length of prompts
        plus the number of tokens we would like to generate
      - pad all the sequences to this length so we can convert them
        into a 2D tensor.
      """
  
      # Tokenize all the prompts.
      tokenizer = get_tokenizer()
      if add_BOS:
          prompts_tokens = [
              [tokenizer.eod] + tokenizer.tokenize(prompt)
              for prompt in prompts
          ]
      else:
          prompts_tokens = [
              tokenizer.tokenize(prompt) for prompt in prompts
          ]
  
      # Now we have a list of list of tokens which each list has a different
      # size. We want to extend this list to:
      #   - incorporate the tokens that need to be generated
      #   - make all the sequences equal length.
      # Get the prompts length.
      prompts_length = [
          len(prompt_tokens) for prompt_tokens in prompts_tokens
      ]
      # Get the max prompts length.
      max_prompt_len = max(prompts_length)
      # Set the tokens to generate to the max prompts length for Retro
      args = get_args()
      if args.retro_add_retriever:
          tokens_to_generate = max_prompt_len
      # Number of tokens in the each sample of the batch.
      samples_length = max_prompt_len + tokens_to_generate
      # Now update the list of list to be of the same size: samples_length.
      for prompt_tokens, prompt_length in zip(
          prompts_tokens, prompts_length
      ):
          padding_size = samples_length - prompt_length
          prompt_tokens.extend([tokenizer.eod] * padding_size)
  
      # Now we are in a structured format, we can convert to tensors.
      prompts_tokens_tensor = torch.cuda.LongTensor(prompts_tokens)
      prompts_length_tensor = torch.cuda.LongTensor(prompts_length)
  
      return prompts_tokens_tensor, prompts_length_tensor
  
file_path: megatron-lm/tools/retro/text_generation/retro_api.py
imports: import numpy
  import torch
  from megatron.core import mpu
  from megatron import print_rank_0, get_retro_args, get_args, get_tokenizer
  from megatron.text_generation.communication import broadcast_float_list, broadcast_tensor, broadcast_int_list
  from megatron.text_generation.generation import score_and_return_on_first_stage
  from tools.retro.text_generation.retro_generation import retro_generate_tokens_probs_and_return_on_first_stage
  from megatron.text_generation.tokenization import detokenize_generations
label_desc: Use image node_id 2 for calling a global function with example usage: _tokenize_prompts_and_batch(prompts, tokens_to_generate, add_BOS) and returns: prompts_tokens_tensor,  prompts_length_tensor