code: def main():
      parser = argparse.ArgumentParser()
      parser.add_argument("--user_input", action="store_true")
      parser.add_argument(
          "--generation_method",
          "-g",
          type=str,
          default="topk",
          choices=["topk", "topp", "ppl"],
          help="generation method",
      )
      args = parser.parse_args()
  
      print("initializing gpt tokenizer...")
  
      ls_tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
      # lightseq use len(tokenizer) as pad_token in default
      ls_tokenizer.add_special_tokens({"pad_token": "[PAD]"})
      print(
          f"lightseq tokenizer pad token id: {ls_tokenizer.pad_token_id}"
      )
  
      hf_tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
      # use EOS as PAD for huggingface to avoid warning according to https://huggingface.co/blog/how-to-generate while avoid reshaping the model embedding
      hf_tokenizer.pad_token = hf_tokenizer.eos_token
      print(
          f"huggingface tokenizer pad token id: {hf_tokenizer.pad_token_id}"
      )
  
      print("creating lightseq model...")
      ls_model = lsi.Gpt("lightseq_gpt2_base.hdf5", max_batch_size=16)
  
      print("creating huggingface model...")
      hf_model = GPT2LMHeadModel.from_pretrained("gpt2")
      hf_model.to("cuda:0")
      hf_model.eval()
  
      # lightseq gpt perplexity supports batch infer with different lengths,
      # but sampling doesn't support
      sentences = [
          "I love you, but you say that",
          "I love you, but you say that",
          "I love you, but you say that",
          "I love you, but you say that",
      ]
  
      print("====================START warmup====================")
      warmup(
          ls_tokenizer,
          hf_tokenizer,
          ls_model,
          hf_model,
          sentences,
          args.generation_method,
      )
      print("====================END warmup====================")
  
      while True:
          if args.user_input:
              sentences = [input("input the masked sentence:\n")]
  
          print("tokenizing the sentences...")
  
          ls_inputs = ls_tokenizer(
              sentences, return_tensors="pt", padding=True
          )["input_ids"]
          hf_inputs = hf_tokenizer(
              sentences, return_tensors="pt", padding=True
          )["input_ids"]
  
          if (
              args.generation_method == "topk"
              or args.generation_method == "topp"
          ):
              ls_generate(ls_model, ls_tokenizer, ls_inputs)
              hf_generate(hf_model, hf_tokenizer, hf_inputs)
          elif args.generation_method == "ppl":
              ls_ppl(ls_model, ls_tokenizer, ls_inputs)
              hf_ppl(hf_model, hf_tokenizer, hf_inputs)
  
          if not args.user_input:
              break
  
file_path: lightseq/examples/inference/python/test/ls_gpt2.py
imports: import time
  import argparse
  import torch
  import lightseq.inference
  from transformers import GPT2Tokenizer, GPT2LMHeadModel
label_desc: Use image node_id 9 for calling a global function with example usage: main() without return types